services:
  vllm:
    image: vllm/vllm-openai:latest
    container_name: vllm
    network_mode: bridge
    environment:
      - HF_TOKEN=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
      - VLLM_GPU_MEMORY_UTILIZATION=0.97
      - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0
    ipc: host
    command: >
      --model Qwen/Qwen2.5-14B-Instruct-AWQ 
      --max-model-len 8192
      --gpu-memory-utilization 0.97
    ports:
      - "8000:8000"
    volumes:
      - /Var/lib/docker/volumes/vllm/huggingface:/root/.cache/huggingface
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [ gpu ]
    runtime: nvidia
    # command: --model NVFP4/Qwen3-Coder-30B-A3B-Instruct-FP4