diff --git a/composte-hermes.yml b/composte-hermes.yml index 7b6ce11..857c0b3 100644 --- a/composte-hermes.yml +++ b/composte-hermes.yml @@ -21,19 +21,17 @@ services: count: 2 capabilities: [gpu] command: - - --model=Qwen/Qwen3-14B-AWQ - - --port=8005 - - --quantization=awq - - --tensor-parallel-size=2 - - --gpu-memory-utilization=0.82 - - --max-model-len=65536 - - --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}} - - --max-num-seqs=16 - - --kv-cache-dtype=fp8 - - --served-model-name=qwen3-14b-awq - - --enable-auto-tool-choice - - --tool-call-parser=hermes - - --reasoning-parser=deepseek_r1 + - --model=Qwen/Qwen3-14B-AWQ + - --port=8005 + - --quantization=awq + - --tensor-parallel-size=2 + - --gpu-memory-utilization=0.84 + - --max-model-len=65536 + - --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}} + - --max-num-seqs=4 + - --served-model-name=qwen3-14b-awq + - --enable-auto-tool-choice + - --tool-call-parser=hermes healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8005/health"] interval: 30s