marc
/
BrainDock
Template
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

compose-vLLM.yaml aktualisiert

This commit is contained in:
Marc Blatter 2026-07-10 18:37:14 +00:00
parent 038f30bd6c
commit 6b309f8afa
1 changed files with 6 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
compose-vLLM.yaml
View File

@ -3,20 +3,17 @@ services:
image: vllm/vllm-openai:latest image: vllm/vllm-openai:latest
container_name: vllm container_name: vllm
restart: unless-stopped restart: unless-stopped
ipc: host # vLLM braucht shared memory für Tensor-Parallelism ipc: host
ports: ports:
- "8000:8000" # OpenAI-kompatible API - "8000:8000"
volumes: volumes:
- /opt/vLLM/models:/root/.cache/huggingface # HF-Model-Cache - /opt/vLLM/models:/root/.cache/huggingface
- /opt/vLLM/config:/vllm-workspace/config - /opt/vLLM/config:/vllm-workspace/config
environment: environment:
- HUGGING_FACE_HUB_TOKEN=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 - HUGGING_FACE_HUB_TOKEN=hf_12345
- NVIDIA_VISIBLE_DEVICES=all - NVIDIA_VISIBLE_DEVICES=all
- NVIDIA_DRIVER_CAPABILITIES=compute,utility - NVIDIA_DRIVER_CAPABILITIES=compute,utility
- VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
deploy: deploy:
resources: resources:
reservations: reservations:
@ -24,12 +21,12 @@ services:
- driver: nvidia - driver: nvidia
count: all count: all
capabilities: [gpu] capabilities: [gpu]
command: > command: >
--model Qwen/Qwen2.5-7B-Instruct-AWQ --model Qwen/Qwen2.5-7B-Instruct-AWQ
--host 0.0.0.0 --host 0.0.0.0
--port 8000 --port 8000
--gpu-memory-utilization 0.85 --gpu-memory-utilization 0.85
--max-model-len 65536 --max-model-len 65536
--hf-overrides {"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}}
--enable-auto-tool-choice --enable-auto-tool-choice
--tool-call-parser hermes --tool-call-parser hermes