marc
/
Cortex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Neue Korrektur mit ChatGPT

This commit is contained in:
Marc Blatter 2026-07-26 17:46:59 +00:00
parent 7a46d61e50
commit c117a5561d
1 changed files with 11 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
composte-hermes.yml
View File

@ -21,19 +21,17 @@ services:
count: 2 count: 2
capabilities: [gpu] capabilities: [gpu]
command: command:
- --model=Qwen/Qwen3-14B-AWQ - --model=Qwen/Qwen3-14B-AWQ
- --port=8005 - --port=8005
- --quantization=awq - --quantization=awq
- --tensor-parallel-size=2 - --tensor-parallel-size=2
- --gpu-memory-utilization=0.82 - --gpu-memory-utilization=0.84
- --max-model-len=65536 - --max-model-len=65536
- --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}} - --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}}
- --max-num-seqs=16 - --max-num-seqs=4
- --kv-cache-dtype=fp8 - --served-model-name=qwen3-14b-awq
- --served-model-name=qwen3-14b-awq - --enable-auto-tool-choice
- --enable-auto-tool-choice - --tool-call-parser=hermes
- --tool-call-parser=hermes
- --reasoning-parser=deepseek_r1
healthcheck: healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8005/health"] test: ["CMD", "curl", "-f", "http://localhost:8005/health"]
interval: 30s interval: 30s