bing
80eef4ce6a
Three always-running vLLM services (text TP=2 GPU0+1, ocr + embed on GPU2, sleep mode) behind a FastAPI router that auto-wakes models on request. Tiered idle (sleep 15 min / offload 3 h), depth-aware 503s with Retry-After, persisted wake-intent recovery, admin API on 127.0.0.1:8010. Routine control via vllmctl is pure HTTP — no docker on the request path. Verified: 91 router unit tests + 15-test E2E on real hardware (measurements in CALIBRATION.md; design record in .claude/memory/router-front-door-plan.md). Old nginx stack files removed before git init; design survives in .claude/memory/sleep-mode-implementation-plan.md. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
96 lines
3.8 KiB
Python
96 lines
3.8 KiB
Python
"""Single-flight wake: N concurrent requests -> exactly one wake sequence."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
from config import DEPTH_AWAKE
|
|
|
|
|
|
async def test_ten_concurrent_requests_trigger_one_wake(pub, backend):
|
|
backend.set_sleeping("text", True, level=2)
|
|
backend.services["vllm-text"]["wake_delay"] = 0.15
|
|
|
|
async def one(i: int):
|
|
r = await pub.post("/v1/chat/completions", json={"model": "text", "seed": i})
|
|
assert r.status_code == 200, r.text
|
|
return r.json()
|
|
|
|
results = await asyncio.gather(*(one(i) for i in range(10)))
|
|
assert all(r["service"] == "vllm-text" for r in results)
|
|
|
|
svc = backend.services["vllm-text"]
|
|
assert svc["wake_up_calls"] == 1, backend.calls
|
|
assert svc["reload_calls"] == 1
|
|
assert svc["api_calls"] == 10
|
|
|
|
|
|
async def test_hold_timeout_does_not_cancel_the_wake(pub, backend, stack):
|
|
"""The first caller times out; the wake keeps going and finishes for the
|
|
next caller (no second wake sequence, no half-awake proxy)."""
|
|
backend.set_sleeping("text", True)
|
|
backend.services["vllm-text"]["wake_delay"] = 0.3
|
|
stack.cfg.hold_offload_s = 0.05
|
|
|
|
r = await pub.post("/v1/chat/completions", json={"model": "text"})
|
|
assert r.status_code == 503
|
|
assert r.json()["error"]["sleep_depth"] == "offloaded"
|
|
|
|
await asyncio.sleep(0.5) # let the shielded wake task finish
|
|
svc_state = stack.manager.services["text"]
|
|
assert svc_state.depth == DEPTH_AWAKE
|
|
assert backend.services["vllm-text"]["wake_up_calls"] == 1
|
|
|
|
r = await pub.post("/v1/chat/completions", json={"model": "text"})
|
|
assert r.status_code == 200
|
|
assert backend.services["vllm-text"]["wake_up_calls"] == 1 # cached awake
|
|
|
|
|
|
async def test_two_services_wake_concurrently_without_cross_talk(pub, backend):
|
|
backend.set_sleeping("text", True)
|
|
backend.set_sleeping("embed", True)
|
|
backend.services["vllm-text"]["wake_delay"] = 0.2
|
|
backend.services["vllm-embed"]["wake_delay"] = 0.05
|
|
|
|
chat = pub.post("/v1/chat/completions", json={"model": "text"})
|
|
embed = pub.post("/v1/embeddings", json={"input": "hi"})
|
|
r1, r2 = await asyncio.gather(chat, embed)
|
|
assert r1.status_code == 200 and r1.json()["service"] == "vllm-text"
|
|
assert r2.status_code == 200 and r2.json()["service"] == "vllm-embed"
|
|
assert backend.services["vllm-text"]["wake_up_calls"] == 1
|
|
assert backend.services["vllm-embed"]["wake_up_calls"] == 1
|
|
|
|
|
|
async def test_wake_state_is_cached_between_requests(pub, backend, stack):
|
|
stack.cfg.state_cache_ttl = 30.0
|
|
backend.set_sleeping("ocr", True)
|
|
assert (await pub.post("/v1/chat/completions", json={"model": "ocr"})).status_code == 200
|
|
probes_after_first = backend.count("GET vllm-ocr/is_sleeping")
|
|
assert probes_after_first >= 1
|
|
for _ in range(5):
|
|
assert (await pub.post("/v1/chat/completions", json={"model": "ocr"})).status_code == 200
|
|
# no extra probe per request while the awake state is cached
|
|
assert backend.count("GET vllm-ocr/is_sleeping") == probes_after_first
|
|
|
|
|
|
async def test_expired_cache_reprobes(pub, backend, stack):
|
|
stack.cfg.state_cache_ttl = 0.0
|
|
for _ in range(3):
|
|
assert (await pub.post("/v1/chat/completions", json={"model": "ocr"})).status_code == 200
|
|
assert backend.count("GET vllm-ocr/is_sleeping") >= 3
|
|
|
|
|
|
async def test_admin_and_request_path_share_one_lock(adm, pub, backend):
|
|
"""A wake driven from the admin port is joined by the public request path."""
|
|
backend.set_sleeping("text", True)
|
|
backend.services["vllm-text"]["wake_delay"] = 0.2
|
|
|
|
admin_wake = asyncio.create_task(adm.post("/admin/wake/text"))
|
|
await asyncio.sleep(0.05)
|
|
r = await pub.post("/v1/chat/completions", json={"model": "text"})
|
|
admin_result = await admin_wake
|
|
|
|
assert r.status_code == 200
|
|
assert admin_result.status_code == 200
|
|
assert backend.services["vllm-text"]["wake_up_calls"] == 1
|