marc
/
Cortex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

composte-hermes.yml aktualisiert

This commit is contained in:
Marc Blatter 2026-07-26 13:54:06 +00:00
parent a7dd4d6e69
commit 7a46d61e50
1 changed files with 14 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
composte-hermes.yml
View File

@ -11,7 +11,6 @@ services:
- NVIDIA_VISIBLE_DEVICES=0,1 - NVIDIA_VISIBLE_DEVICES=0,1
- HUGGING_FACE_HUB_TOKEN=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 - HUGGING_FACE_HUB_TOKEN=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
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
- VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
volumes: volumes:
- /opt/vllm/huggingface:/root/.cache/huggingface - /opt/vllm/huggingface:/root/.cache/huggingface
deploy: deploy:
@ -22,17 +21,19 @@ services:
count: 2 count: 2
capabilities: [gpu] capabilities: [gpu]
command: command:
- --model=Qwen/Qwen3-14B-AWQ - --model=Qwen/Qwen3-14B-AWQ
- --tensor-parallel-size=2 - --port=8005
- --gpu-memory-utilization=0.92 - --quantization=awq
- --max-model-len=65536 - --tensor-parallel-size=2
- --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":40960}} - --gpu-memory-utilization=0.82
- --enable-prefix-caching - --max-model-len=65536
- --enable-auto-tool-choice - --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}}
- --tool-call-parser=hermes - --max-num-seqs=16
- --max-num-seqs=4 - --kv-cache-dtype=fp8
- --generation-config=vllm - --served-model-name=qwen3-14b-awq
- --disable-custom-all-reduce - --enable-auto-tool-choice
- --tool-call-parser=hermes
- --reasoning-parser=deepseek_r1
healthcheck: healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8005/health"] test: ["CMD", "curl", "-f", "http://localhost:8005/health"]
interval: 30s interval: 30s
@ -57,6 +58,7 @@ services:
volumes: volumes:
- /opt/hermes:/opt/data - /opt/hermes:/opt/data
- /opt/hermes/.ssh:/root/.ssh - /opt/hermes/.ssh:/root/.ssh
- /var/run/docker.sock:/var/run/docker.sock
environment: environment:
- OPENAI_BASE_URL=http://vllm:8005/v1 - OPENAI_BASE_URL=http://vllm:8005/v1