PVE GPU 透传实战：4 卡 RTX 3090 与 vLLM 推理

# 假设目标磁盘为 /dev/sda，使用 fdisk 分区：
# sda1: EFI (FAT32)
# sda2: Boot (Ext4)
# sda3: Root (Ext4)

apt install -y dosfstools cryptsetup efibootmgr debootstrap

mkfs.fat -F32 /dev/sda1
mkfs.ext4 /dev/sda2
mkfs.ext4 /dev/sda3

# 挂载目录结构
mkdir -pv /mnt/debinst
mount /dev/sda3 /mnt/debinst/
mkdir -pv /mnt/debinst/boot
mount /dev/sda2 /mnt/debinst/boot/
mkdir -pv /mnt/debinst/boot/efi
mount /dev/sda1 /mnt/debinst/boot/efi/

debootstrap --arch amd64 trixie /mnt/debinst https://mirrors.tuna.tsinghua.edu.cn/debian/

# 绑定必要的核心文件系统
mount -v -t proc none /mnt/debinst/proc
mount -v --rbind /sys /mnt/debinst/sys
mount -v --rbind /dev /mnt/debinst/dev
mount -v --make-rslave /mnt/debinst/sys
mount -v --make-rslave /mnt/debinst/dev

# 切换到真实的 VM 环境
LANG=C.UTF-8 chroot /mnt/debinst /bin/bash

apt update
apt install -y apt-transport-https ca-certificates aptitude vim dosfstools efibootmgr lvm2

# 填入刚装好的系统分区 UUID（可通过 blkid 获取）
cat <<EOF > /etc/fstab
UUID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /          ext4 defaults,relatime 0 1
UUID=xxxx-xxxx                            /boot/efi  vfat defaults 0 1
UUID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /boot      ext4 defaults,relatime 0 1
EOF

# 主机名与 DNS
echo v100-cluster > /etc/hostname
echo nameserver 114.114.114.114 > /etc/resolv.conf    

# 静态网络配置
cat <<EOF > /etc/network/interfaces
auto ens18
iface ens18 inet static
    address 192.168.1.21
    network 192.168.1.0
    netmask 255.255.255.0
    broadcast 192.168.1.255
    gateway 192.168.1.1
EOF

cat <<EOF > /etc/apt/sources.list
deb https://mirrors.tuna.tsinghua.edu.cn/debian/ trixie main contrib non-free non-free-firmware
deb https://mirrors.tuna.tsinghua.edu.cn/debian/ trixie-updates main contrib non-free non-free-firmware
deb https://mirrors.tuna.tsinghua.edu.cn/debian/ trixie-backports main contrib non-free non-free-firmware
deb https://mirrors.tuna.tsinghua.edu.cn/debian-security trixie-security main contrib non-free non-free-firmware
EOF

aptitude update
aptitude install locales tzdata
dpkg-reconfigure locales
dpkg-reconfigure tzdata

apt-get install linux-image-amd64

# 必须安装 EFI 引导程序
aptitude install grub-efi
grub-install --target=x86_64-efi --efi-directory=/boot/efi --bootloader-id=Proxmox
grub-mkconfig -o /boot/grub/grub.cfg

# 开启 root用户的 SSH 登录，对于无头服务器环境非常重要
apt-get install ssh -y
sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
passwd

exit
umount -R /mnt/debinst
reboot

apt install -y python3-venv python3-pip

python3 -m venv gpu_test_env
source gpu_test_env/bin/activate

# 镜像加速
python -m pip install -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple --upgrade pip
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

# 只安装核心计算库
pip install torch torchvision torchaudio 

# 测试 GPU 是否被 PyTorch 识别
python -c "import torch; print(f'CUDA 核心可用: {torch.cuda.is_available()}'); print(f'GPU 数量: {torch.cuda.device_count()}')"

nvidia-smi topo -m
	GPU0	GPU1	GPU2	GPU3	GPU4	GPU5	GPU6	GPU7	CPU Affinity	NUMA Affinity	GPU NUMA ID
GPU0	 X 	NV2	NV2	NV1	PHB	PHB	PHB	NV1	0-39	0		N/A
GPU1	NV2	 X 	NV1	NV2	PHB	PHB	NV1	PHB	0-39	0		N/A
...

nvidia-smi nvlink -s
GPU 0: Tesla V100-SXM2-32GB (UUID: GPU-4a303885-0edf-7219-d89d-f863a9e11366)
	 Link 0: 25.781 GB/s
	 Link 1: 25.781 GB/s
...

apt install -y git cmake libboost-program-options-dev nvidia-cuda-toolkit build-essential

# 编译最新的 NVIDIA NVBandwidth 工具
git clone https://github.com/NVIDIA/nvbandwidth.git
cd nvbandwidth
cmake . && make

# Host-to-Device / Device-to-Host 带宽
./nvbandwidth -t host_to_device_memcpy_ce
./nvbandwidth -t device_to_host_memcpy_ce

# GPU 间 P2P 带宽 (Device-to-Device)
./nvbandwidth -t device_to_device_memcpy_ce

    Product Name                          : NVIDIA GeForce RTX 3090
    GPU UUID                              : GPU-1dc002ea-dc6e-5806-d798-43de15b92028
    FB Memory Usage
    BAR1 Memory Usage
    Conf Compute Protected Memory Usage
        Memory                            : 0 %
        Memory Current Temp               : N/A
        Memory Max Operating Temp         : N/A
    GPU Memory Power Readings
        Memory                            : 405 MHz
        Memory                            : N/A
        Memory                            : N/A
        Memory                            : N/A
        Memory                            : 9751 MHz
    Product Name                          : NVIDIA GeForce RTX 3090
    GPU UUID                              : GPU-5b7a34da-4cbe-86e8-9607-e6d2b336b944
    FB Memory Usage
    BAR1 Memory Usage
    ... (其余 3 张卡同上)

    Product Name                          : Tesla V100-SXM2-32GB
    GPU UUID                              : GPU-4a303885-0edf-7219-d89d-f863a9e11366
    FB Memory Usage
    BAR1 Memory Usage
    Conf Compute Protected Memory Usage
        Memory                            : 0 %
                Device Memory             : 0
                Texture Memory            : N/A
                Device Memory             : 0
                Texture Memory            : N/A
                ... 
        Memory Current Temp               : 37 C
        Memory Max Operating Temp         : 85 C
    GPU Memory Power Readings
        Memory                            : 877 MHz
        ...

# train_benchmark.py
import torch, time, argparse, os
import torch.nn as nn
import torch.optim as optim
import torchvision.models as models

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--model', default='resnet50')
    parser.add_argument('--batch-size', type=int, default=128)
    parser.add_argument('--epochs', type=int, default=5)
    parser.add_argument('--num-gpus', type=int, default=1)
    parser.add_argument('--local_rank', type=int, default=-1)
    args = parser.parse_args()

    local_rank = int(os.environ.get("LOCAL_RANK", args.local_rank))
    is_ddp = local_rank != -1

    if is_ddp:
        torch.distributed.init_process_group(backend='nccl')
        torch.cuda.set_device(local_rank)
        device = torch.device(f"cuda:{local_rank}")
    else:
        device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    model = models.resnet50().to(device)
    if is_ddp:
        model = nn.parallel.DistributedDataParallel(model, device_ids=[local_rank])

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(model.parameters(), lr=0.01)
    inputs = torch.randn(args.batch_size, 3, 224, 224).to(device)
    targets = torch.randint(0, 1000, (args.batch_size,)).to(device)

    for epoch in range(args.epochs):
        torch.cuda.synchronize()
        start = time.time()
        for _ in range(50):
            optimizer.zero_grad()
            loss = criterion(model(inputs), targets)
            loss.backward()
            optimizer.step()
        torch.cuda.synchronize()
        
        if local_rank in [-1, 0]:
            throughput = (50 * args.batch_size * (args.num_gpus if is_ddp else 1)) / (time.time() - start)
            print(f"Epoch {epoch} Throughput: {throughput:.2f} images/sec")

if __name__ == '__main__':
    main()

# 测试节点临时环境使用 pip 部署即可
# 1. 切换 PyPI 清华源加速安装
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
pip install vllm

# 2. 注入 HF 国内镜像站环境变量，防止起服动态拉取模型时卡死超时
export HF_ENDPOINT="https://hf-mirror.com"


# 9B 2卡
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen3.5-9B \
    --dtype float16 \
    --tensor-parallel-size 2 \
    --gpu-memory-utilization 0.9 \
    --max-model-len 4096 \
    --port 8000


# 9B 4卡
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen3.5-9B \
    --dtype float16 \
    --tensor-parallel-size 4 \
    --gpu-memory-utilization 0.9 \
    --max-model-len 4096 \
    --port 8000

# 多卡 TP=4，跑 122B MoE 模型
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen3.5-122B-A10B \
    --dtype float16 \
    --tensor-parallel-size 4 \
    --gpu-memory-utilization 0.9 \
    --max-model-len 4096 \
    --port 8000

global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'dcgm-exporter'
    scrape_interval: 5s      # GPU 指标用更短的采集间隔
    static_configs:
      - targets: ['localhost:9400']

  - job_name: 'node-exporter'
    static_configs:
      - targets: ['localhost:9100']

  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

# Prometheus
docker run -d --restart always \
    --name prometheus \
    --net="host" \
    -v /opt/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml \
    -v /opt/monitoring/prometheus/data:/prometheus \
    prom/prometheus:latest \
    --config.file=/etc/prometheus/prometheus.yml \
    --storage.tsdb.path=/prometheus \
    --storage.tsdb.retention.time=30d \
    --web.enable-lifecycle

# Grafana
mkdir -p /opt/monitoring/grafana
docker run -d --restart always \
    --name grafana \
    --net="host" \
    -v /opt/monitoring/grafana:/var/lib/grafana \
    -e "GF_SECURITY_ADMIN_PASSWORD=admin" \
    grafana/grafana:latest

# 温度趋势
DCGM_FI_DEV_GPU_TEMP{job="dcgm-exporter"}

# SM 降频观察
DCGM_FI_DEV_SM_CLOCK{job="dcgm-exporter"}

# 功耗墙触发频率
rate(DCGM_FI_DEV_POWER_VIOLATION{job="dcgm-exporter"}[1m])

# XID 错误（任何非零都是红灯）
DCGM_FI_DEV_XID_ERRORS{job="dcgm-exporter"}

# PCIe Replay 错误趋势
DCGM_FI_DEV_PCIE_REPLAY_COUNTER{job="dcgm-exporter"}

# /opt/monitoring/docker-compose.yml
version: '3.8'

services:
  dcgm-exporter:
    image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-debian12
    restart: always
    deploy:
      resources:
        reservations:
          devices:
            - capabilities: [gpu]
    ports:
      - "9400:9400"

  node-exporter:
    image: quay.io/prometheus/node-exporter:latest
    restart: always
    network_mode: host
    pid: host
    volumes:
      - /:/host:ro,rslave
    command:
      - '--path.rootfs=/host'

  prometheus:
    image: prom/prometheus:latest
    restart: always
    network_mode: host
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
      - '--web.enable-lifecycle'

  grafana:
    image: grafana/grafana:latest
    restart: always
    network_mode: host
    volumes:
      - ./grafana:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin

# 实时观察
watch -n 1 nvidia-smi

# 结构化日志记录
while true; do
    echo "=== $(date) ==="
    nvidia-smi --query-gpu=timestamp,name,gpu_bus_id,temperature.gpu,utilization.gpu,utilization.memory,power.draw,clocks.sm,clocks.mem,memory.used,memory.total,pcie.link.gen.current,pcie.link.width.current --format=csv
    sleep 5
done | tee gpu_burn_log_$(date +%Y%m%d_%H%M%S).csv

# DCGM 完整诊断（在烤机结束后跑一次）
dcgmi diag -r 3

# 安装 HF 官方工具包
pip install -U "huggingface_hub[cli]"

# 环境变量注入
export HF_TOKEN="hf_xxxxx_your_token_xxxxx"

# 使用国内镜像站加速（避免 huggingface.co 连接超时）
export HF_ENDPOINT="https://hf-mirror.com"

# 指定模型下载的统一目录（比如挂载的高速 NVMe 盘）
export HF_HOME="/data/llm/models"

# 使用 CLI 高速下载（支持断点续传）
# Llama-3.1 是受限模型，必须有 HF_TOKEN 才能拉取
hf download Qwen/Qwen3.5-9B
hf download Qwen/Qwen3.5-35B-A3B
hf download Qwen/Qwen3-Coder-Next
hf download zai-org/GLM-5
hf  download MiniMaxAI/MiniMax-M2.5
hf  download Qwen/Qwen3.5-122B-A10B

# 使用官方安装脚本，通过清华镜像源加速
export DOWNLOAD_URL="https://mirrors.tuna.tsinghua.edu.cn/docker-ce"
curl -fsSL https://get.docker.com | sh

# 安装 NVIDIA Container Toolkit（让 Docker 能访问透传的 GPU）
apt-get update &&  apt-get install -y --no-install-recommends \
   ca-certificates \
   curl \
   gnupg2

curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey |  gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
  && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
     tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

export NVIDIA_CONTAINER_TOOLKIT_VERSION=1.18.2-1
   apt-get install -y \
      nvidia-container-toolkit=${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      nvidia-container-toolkit-base=${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      libnvidia-container-tools=${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      libnvidia-container1=${NVIDIA_CONTAINER_TOOLKIT_VERSION}


nvidia-ctk runtime configure --runtime=docker
systemctl restart docker

# 验证 GPU 在容器内可见
docker run --rm --gpus all hub.infra.plz.ac/nvidia/cuda:12.4.0-base-ubuntu22.04 nvidia-smi

# /opt/llm-inference/docker-compose.yml
services:
  vllm-fast:
    image: vllm/vllm-openai:latest
    restart: always
    ipc: host
    ports:
      - "127.0.0.1:8001:8000"
    volumes:
      - /data/llm/models:/root/.cache/huggingface
    environment:
      - HF_ENDPOINT=https://hf-mirror.com
      - TRANSFORMERS_OFFLINE=1
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ['0']
              capabilities: [gpu]
    command: >
      --model Qwen/Qwen3.5-9B
      --dtype float16
      --gpu-memory-utilization 0.9
      --max-model-len 4096
      --port 8000
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3

  vllm-balanced:
    image: vllm/vllm-openai:latest
    restart: always
    ipc: host
    ports:
      - "127.0.0.1:8002:8000"
    volumes:
      - /data/llm/models:/root/.cache/huggingface
    environment:
      - HF_ENDPOINT=https://hf-mirror.com
      - TRANSFORMERS_OFFLINE=1
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ['1']
              capabilities: [gpu]
    command: >
      --model Qwen/Qwen3.5-35B-A3B
      --dtype float16
      --gpu-memory-utilization 0.9
      --max-model-len 4096
      --port 8000
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3

  vllm-reasoning:
    image: vllm/vllm-openai:latest
    restart: always
    ipc: host
    ports:
      - "127.0.0.1:8003:8000"
    volumes:
      - /data/llm/models:/root/.cache/huggingface
    environment:
      - HF_ENDPOINT=https://hf-mirror.com
      - TRANSFORMERS_OFFLINE=1
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ['2', '3']
              capabilities: [gpu]
    command: >
      --model Qwen/Qwen3.5-122B-A10B
      --dtype float16
      --tensor-parallel-size 2
      --gpu-memory-utilization 0.9
      --max-model-len 8192
      --port 8000
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3

# /opt/llm-gateway/litellm_config.yaml
model_list:
  - model_name: "fast"
    litellm_params:
      model: "openai/meta-llama/Llama-3.1-8B-Instruct"
      api_base: "http://127.0.0.1:8001/v1"
      api_key: "not-used"

  - model_name: "balanced"
    litellm_params:
      model: "openai/Qwen/Qwen2.5-14B-Chat-GPTQ-Int4"
      api_base: "http://127.0.0.1:8002/v1"
      api_key: "not-used"

  - model_name: "reasoning"
    litellm_params:
      model: "openai/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
      api_base: "http://127.0.0.1:8003/v1"
      api_key: "not-used"

  # 外部 Fallback：本地 reasoning 不可用时自动降级
  - model_name: "reasoning"
    litellm_params:
      model: "claude-sonnet-4-20250514"
      api_key: "${ANTHROPIC_API_KEY}"

router_settings:
  routing_strategy: "simple-shuffle"
  num_retries: 2
  timeout: 300
  allowed_fails: 3               # 连续失败 3 次暂时摘除后端

general_settings:
  master_key: "${LITELLM_MASTER_KEY}"
  database_url: "sqlite:///litellm.db"
  allow_user_auth: true

litellm_settings:
  max_budget: 100
  budget_duration: "monthly"
  success_callback: ["langfuse"]  # 请求级 tracing
  store_model_in_db: true         # 审计日志持久化

# OpenClaw Agent — 三个模型都可用，月预算 $50
curl -X POST http://localhost:4000/key/generate \
  -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \
  -d '{
    "key_alias": "openclaw-agent",
    "models": ["fast", "balanced", "reasoning"],
    "max_budget": 50, "budget_duration": "monthly",
    "max_parallel_requests": 5,
    "tpm_limit": 100000, "rpm_limit": 60
  }'

# Web 应用 — 只允许 fast，月预算 $10
curl -X POST http://localhost:4000/key/generate \
  -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \
  -d '{
    "key_alias": "webapp-chatbot",
    "models": ["fast"],
    "max_budget": 10, "budget_duration": "monthly",
    "max_parallel_requests": 10,
    "tpm_limit": 50000, "rpm_limit": 100
  }'

server {
    listen 443 ssl;
    server_name llm.internal;

    ssl_certificate     /etc/ssl/llm-gateway.crt;
    ssl_certificate_key /etc/ssl/llm-gateway.key;
    ssl_client_certificate /etc/ssl/ca.crt;
    ssl_verify_client on;

    location / {
        proxy_pass http://127.0.0.1:4000;
        proxy_read_timeout 600s;
    }
}

# docker-compose.langfuse.yml
version: '3.8'
services:
  langfuse-server:
    image: langfuse/langfuse:2
    depends_on:
      - db
    ports:
      - "3000:3000"
    environment:
      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
      - NEXTAUTH_SECRET=my_secure_secret
      - SALT=my_secure_salt
      - NEXTAUTH_URL=http://192.168.1.14:3000
      - TELEMETRY_ENABLED=false
  db:
    image: postgres:15
    environment:
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=postgres
      - POSTGRES_DB=postgres
    volumes:
      - langfuse_data:/var/lib/postgresql/data

volumes:
  langfuse_data:

# serve_app.py
from ray import serve
from vllm import LLM, SamplingParams

@serve.deployment(num_replicas=1, ray_actor_options={"num_gpus": 1})
class LLMDeployment:
    def __init__(self, model_name: str):
        self.llm = LLM(model=model_name, dtype="float16", gpu_memory_utilization=0.9)

    async def generate(self, prompt: str, max_tokens: int = 512):
        params = SamplingParams(max_tokens=max_tokens)
        outputs = self.llm.generate([prompt], params)
        return outputs[0].outputs[0].text

light = LLMDeployment.bind("meta-llama/Llama-3.1-8B-Instruct")
heavy = LLMDeployment.bind("deepseek-ai/DeepSeek-R1-Distill-Qwen-32B")

{
  "models": {
    "providers": {
      "gpu-cluster": {
        "type": "openai-compatible",
        "baseUrl": "https://llm.internal/v1",
        "apiKey": "${LITELLM_API_KEY}",
        "models": [
          { "id": "fast",      "name": "快速响应 (8B)",  "contextLength": 4096 },
          { "id": "balanced",  "name": "均衡 (14B)",     "contextLength": 4096 },
          { "id": "reasoning", "name": "深度推理 (32B)", "contextLength": 8192 }
        ]
      }
    }
  },
  "agents": {
    "defaults": {
      "model": "gpu-cluster:fast"
    }
  },
  "tools": {
    "profile": "messaging",
    "deny": ["group:automation", "group:runtime"],
    "exec": { "security": "deny", "ask": "always" },
    "elevated": { "enabled": false }
  },
  "channels": {
    "telegram": {
      "dmPolicy": "pairing",
      "groups": { "*": { "requireMention": true } }
    }
  }
}

<!-- ~/.openclaw/skills/model-router/SKILL.md -->
---
name: model-router
description: 根据任务复杂度自动选择 LLM 模型
---

## 模型选择规则

根据以下规则选择模型，切换前无需询问用户：

1. **简单任务**（翻译、格式转换、日常对话、摘要）→ `gpu-cluster:fast`
2. **中等任务**（技术问答、内容创作、文档编写）→ `gpu-cluster:balanced`
3. **复杂任务**（代码审查、数学推理、架构设计、多步分析）→ `gpu-cluster:reasoning`

切换时使用 `/model` 命令。如果不确定复杂度，默认使用 `balanced`。