marc
/
Cortex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

enable-prefix-caching

This commit is contained in:
Marc Blatter 2026-07-29 09:18:17 +00:00
parent cb32e0b429
commit 6b8a1f341a
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
composte-hermes.yml
View File

@ -32,6 +32,8 @@ services:
- --served-model-name=qwen3-14b-awq - --served-model-name=qwen3-14b-awq
- --enable-auto-tool-choice - --enable-auto-tool-choice
- --tool-call-parser=hermes - --tool-call-parser=hermes
- --enable-prefix-caching # 2x RTX 3060 (12 GB)
- --prefix-cache-threshold=0.8 # 24 GB VRAM
healthcheck: healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8005/health"] test: ["CMD", "curl", "-f", "http://localhost:8005/health"]
interval: 30s interval: 30s