marc
/
BrainDock
Template
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

compose-vLLM.yaml aktualisiert

This commit is contained in:
Marc Blatter 2026-07-10 18:38:43 +00:00
parent 6b309f8afa
commit ae5f21ace5
1 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
compose-vLLM.yaml
View File

@ -21,12 +21,19 @@ services:
- driver: nvidia - driver: nvidia
count: all count: all
capabilities: [gpu] capabilities: [gpu]
command: > command:
--model Qwen/Qwen2.5-7B-Instruct-AWQ - "--model"
--host 0.0.0.0 - "Qwen/Qwen2.5-7B-Instruct-AWQ"
--port 8000 - "--host"
--gpu-memory-utilization 0.85 - "0.0.0.0"
--max-model-len 65536 - "--port"
--hf-overrides {"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}} - "8000"
--enable-auto-tool-choice - "--gpu-memory-utilization"
--tool-call-parser hermes - "0.85"
- "--max-model-len"
- "65536"
- "--hf-overrides"
- '{"rope_scaling": {"rope_type": "yarn", "factor": 2.0, "original_max_position_embeddings": 32768}}'
- "--enable-auto-tool-choice"
- "--tool-call-parser"
- "hermes"