diff --git a/composte-hermes.yml b/composte-hermes.yml index c68ca98..931f843 100644 --- a/composte-hermes.yml +++ b/composte-hermes.yml @@ -24,13 +24,15 @@ services: command: - --model=Qwen/Qwen3-14B-AWQ - --tensor-parallel-size=2 - - --gpu-memory-utilization=0.90 + - --gpu-memory-utilization=0.92 - --max-model-len=65536 - - --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}} + - --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":40960}} - --enable-prefix-caching - --enable-auto-tool-choice - --tool-call-parser=hermes - --max-num-seqs=4 + - --generation-config=vllm + - --disable-custom-all-reduce healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8005/health"] interval: 30s