marc
/
BrainDock
Template
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

compose-vLLM.yaml aktualisiert

This commit is contained in:
Marc Blatter 2026-07-10 18:38:43 +00:00
parent 6b309f8afa
commit ae5f21ace5
1 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
compose-vLLM.yaml
View File

@ -21,12 +21,19 @@ services:
- driver: nvidia
count: all
capabilities: [gpu]
command: >
--model Qwen/Qwen2.5-7B-Instruct-AWQ
--host 0.0.0.0
--port 8000
--gpu-memory-utilization 0.85
--max-model-len 65536
--hf-overrides {"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}}
--enable-auto-tool-choice
--tool-call-parser hermes
command:
- "--model"
- "Qwen/Qwen2.5-7B-Instruct-AWQ"
- "--host"
- "0.0.0.0"
- "--port"
- "8000"
- "--gpu-memory-utilization"
- "0.85"
- "--max-model-len"
- "65536"
- "--hf-overrides"
- '{"rope_scaling": {"rope_type": "yarn", "factor": 2.0, "original_max_position_embeddings": 32768}}'
- "--enable-auto-tool-choice"
- "--tool-call-parser"
- "hermes"