marc
/
Cortex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Korrektur durch ChatGPT

This commit is contained in:
Marc Blatter 2026-07-26 12:32:00 +00:00
parent 42fb0e0e99
commit 78b84ab468
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
composte-hermes.yml
View File

@ -11,6 +11,7 @@ services:
- NVIDIA_VISIBLE_DEVICES=0,1 - NVIDIA_VISIBLE_DEVICES=0,1
- HUGGING_FACE_HUB_TOKEN=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 - HUGGING_FACE_HUB_TOKEN=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
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
- VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
volumes: volumes:
- /opt/vllm/huggingface:/root/.cache/huggingface - /opt/vllm/huggingface:/root/.cache/huggingface
deploy: deploy:
@ -25,6 +26,7 @@ services:
- --tensor-parallel-size=2 - --tensor-parallel-size=2
- --gpu-memory-utilization=0.90 - --gpu-memory-utilization=0.90
- --max-model-len=65536 - --max-model-len=65536
- --hf-overrides={"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":32768}}
- --enable-prefix-caching - --enable-prefix-caching
- --enable-auto-tool-choice - --enable-auto-tool-choice
- --tool-call-parser=hermes - --tool-call-parser=hermes