marc
/
Cortex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

composte-hermes.yml aktualisiert

This commit is contained in:
Marc Blatter 2026-07-22 09:02:34 +00:00
parent 3514e0a3e3
commit 9eed8c758c
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
composte-hermes.yml
View File

@ -26,7 +26,8 @@ services:
- --quantization=awq - --quantization=awq
- --tensor-parallel-size=2 - --tensor-parallel-size=2
- --gpu-memory-utilization=0.82 - --gpu-memory-utilization=0.82
- --max-model-len=32768 - --max-model-len=65536
- --rope-scaling={"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}
- --max-num-seqs=16 - --max-num-seqs=16
- --kv-cache-dtype=fp8 - --kv-cache-dtype=fp8
- --served-model-name=qwen3-14b-awq - --served-model-name=qwen3-14b-awq