marc
/
Cortex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Korrektur durch ChatGPT

This commit is contained in:
Marc Blatter 2026-07-26 12:46:47 +00:00
parent 78b84ab468
commit 63ed4bba6e
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
composte-hermes.yml
View File

@ -24,13 +24,15 @@ services:
command:
- --model=Qwen/Qwen3-14B-AWQ
- --tensor-parallel-size=2
- --gpu-memory-utilization=0.90
- --gpu-memory-utilization=0.92
- --max-model-len=65536
- --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}}
- --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":40960}}
- --enable-prefix-caching
- --enable-auto-tool-choice
- --tool-call-parser=hermes
- --max-num-seqs=4
- --generation-config=vllm
- --disable-custom-all-reduce
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8005/health"]
interval: 30s