Multiple update from copilot

2026-09-07 15:53:19 +02:00
parent d0cef24c4c
commit 62e9be61fc
1 changed files with 7 additions and 3 deletions
@@ -5,10 +5,14 @@ services:
    network_mode: bridge
    environment:
      - HF_TOKEN=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
-#      - PYTORCH_ALLOC_CONF=expandable_segments:True
-#      - VLLM_DEVICE=cpu
+      - VLLM_GPU_MEMORY_UTILIZATION=0.97
+      - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0
    ipc: host
-    command: --model Qwen/Qwen2.5-14B-Instruct-AWQ
+    command: >
+      vllm serve
+      Qwen/Qwen2.5-14B-Instruct-AWQ 
+      --max-model-len 8192
+      --gpu-memory-utilization 0.97
    ports:
      - "8000:8000"
    volumes: