在 Debian 12 上部署 SPDK：编译、验证与 NVMe-oF 实测

# 检查发行版与内核版本
cat /etc/os-release | grep -E 'PRETTY_NAME|VERSION_ID'
uname -r

# 检查 CPU 是否支持 SSE4.2（DPDK 必需）
grep -o 'sse4_2' /proc/cpuinfo | head -1

# 检查 IOMMU 是否已启用（可选）
dmesg | grep -i -E 'DMAR|IOMMU' | head -5

# 检查现有 NVMe 设备（可选）
lspci -nn | grep -i 'Non-Volatile memory'
nvme list 2>/dev/null || echo "无 NVMe 设备或 nvme-cli 未安装"

# 查看当前 hugepage 状态
grep -i huge /proc/meminfo

# 设置 1024 个 2MB hugepage（约 2GB）
sudo sysctl -w vm.nr_hugepages=1024

# 确保 hugetlbfs 已挂载
sudo mkdir -p /dev/hugepages
sudo mount -t hugetlbfs nodev /dev/hugepages 2>/dev/null || true

# 验证
grep HugePages_ /proc/meminfo

# 写入 sysctl 配置
cat <<'EOF' | sudo tee /etc/sysctl.d/60-spdk-hugepages.conf
# SPDK hugepage configuration
vm.nr_hugepages = 2048
EOF

# 写入 fstab 确保 hugetlbfs 开机挂载
grep -q hugetlbfs /etc/fstab || \
    echo 'nodev /dev/hugepages hugetlbfs defaults,mode=1770,gid=0 0 0' | sudo tee -a /etc/fstab

sudo sysctl -p /etc/sysctl.d/60-spdk-hugepages.conf
sudo mkdir -p /dev/hugepages
sudo mount -a

# 验证
grep HugePages_ /proc/meminfo

# Intel CPU
sudo mkdir -p /etc/default/grub.d
echo 'GRUB_CMDLINE_LINUX_DEFAULT="$GRUB_CMDLINE_LINUX_DEFAULT intel_iommu=on iommu=pt"' \
    | sudo tee /etc/default/grub.d/90-iommu.cfg

# AMD CPU（二选一）
# echo 'GRUB_CMDLINE_LINUX_DEFAULT="$GRUB_CMDLINE_LINUX_DEFAULT amd_iommu=on iommu=pt"' \
#     | sudo tee /etc/default/grub.d/90-iommu.cfg

sudo update-grub
sudo reboot

sudo cat > /etc/systemd/system/spdk-tgt.service <<'EOF'
[Unit]
Description=SPDK Target Daemon
After=network.target
Wants=network.target

[Service]
Type=simple
# 核心掩码根据实际 NUMA 拓扑调整
ExecStartPre=/sbin/modprobe vfio-pci
ExecStart=/opt/spdk/build/bin/spdk_tgt \
    -m 0x3 \
    -s 4096 \
    -r /var/tmp/spdk.sock \
    --wait-for-rpc
ExecStartPost=/bin/sleep 1
ExecStartPost=/opt/spdk/scripts/rpc.py -s /var/tmp/spdk.sock framework_start_init
ExecStartPost=/opt/spdk/conf/init-bdevs.sh
# 优雅停止
ExecStop=/opt/spdk/scripts/rpc.py -s /var/tmp/spdk.sock spdk_kill_instance SIGTERM
TimeoutStopSec=30
Restart=on-failure
RestartSec=5
# 安全加固
LimitMEMLOCK=infinity
LimitNOFILE=65536
LimitCORE=infinity

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable --now spdk-tgt
sudo systemctl status spdk-tgt

#!/usr/bin/env bash
# /opt/spdk/conf/init-bdevs.sh
# 启动后通过 JSON-RPC 创建所需的 bdev 与 subsystem

set -euo pipefail

RPC="/opt/spdk/scripts/rpc.py -s /var/tmp/spdk.sock"

# 示例：创建 malloc bdev 用于测试（1024 MB，块大小 4096）
$RPC bdev_malloc_create -b Malloc0 1024 4096

# 示例：创建 NVMe-oF/TCP transport
$RPC nvmf_create_transport -t TCP -u 8192 -m 8 -c 4096

echo "SPDK bdev initialization complete."

cd /opt/spdk

# 1) 创建 1 GiB malloc bdev（block_size = 4096）
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_malloc_create -b Malloc0 1024 4096

# 2) 加载 NBD 内核模块（允许最多 16 个 nbd 设备）
sudo modprobe nbd nbds_max=16

# 3) 将 bdev 映射到 /dev/nbd0
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock nbd_start_disk Malloc0 /dev/nbd0

# 4) 确认设备就绪
lsblk | grep nbd0

# 5) fio 基准测试：4K 随机读写混合，队列深度 32
sudo fio --name=spdk-malloc-verify \
    --filename=/dev/nbd0 \
    --ioengine=libaio \
    --direct=1 \
    --rw=randrw \
    --rwmixread=70 \
    --bs=4k \
    --iodepth=32 \
    --numjobs=1 \
    --time_based \
    --runtime=15 \
    --group_reporting

# 6) 查看 SPDK 侧 IO 统计
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_get_iostat -b Malloc0 | jq

# 7) 清理
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock nbd_stop_disk /dev/nbd0
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_malloc_delete Malloc0

cd /opt/spdk

# 1) 停止 spdk_tgt（如在运行），启动 nvmf_tgt
sudo pkill spdk_tgt 2>/dev/null; sleep 1
sudo build/bin/nvmf_tgt -m 0x3 -s 2048 -r /var/tmp/spdk.sock &
sleep 2

# 2) 创建 TCP transport
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock nvmf_create_transport \
    -t TCP \
    -u 8192 \
    -m 8 \
    -c 4096

# 3) 创建 malloc bdev
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_malloc_create -b Malloc0 1024 4096

# 4) 创建 NVMe-oF subsystem（NQN 需全局唯一）
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock nvmf_create_subsystem \
    nqn.2025-10.io.spdk:cnode1 \
    -a -s SPDK00000001

# 5) 将 bdev 作为 namespace 挂载到 subsystem
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock nvmf_subsystem_add_ns \
    nqn.2025-10.io.spdk:cnode1 Malloc0

# 6) 添加 TCP listener
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock nvmf_subsystem_add_listener \
    nqn.2025-10.io.spdk:cnode1 \
    -t tcp -a 127.0.0.1 -s 4420

# 7) 内核 initiator 端：发现与连接
sudo modprobe nvme-tcp
sudo nvme discover -t tcp -a 127.0.0.1 -s 4420
sudo nvme connect -t tcp -n nqn.2025-10.io.spdk:cnode1 -a 127.0.0.1 -s 4420

# 8) 确认新设备出现
sleep 1
sudo nvme list
ls -la /dev/nvme*n* 2>/dev/null

# 9) fio 验证
NVME_DEV=$(ls /dev/nvme*n1 2>/dev/null | tail -1)
if [ -n "$NVME_DEV" ]; then
    sudo fio --name=nvmeof-tcp-verify \
        --filename="$NVME_DEV" \
        --ioengine=libaio \
        --direct=1 \
        --rw=randread \
        --bs=4k \
        --iodepth=32 \
        --numjobs=1 \
        --time_based \
        --runtime=10 \
        --group_reporting
fi

# 10) 清理（顺序很重要）
sudo nvme disconnect -n nqn.2025-10.io.spdk:cnode1
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock nvmf_delete_subsystem nqn.2025-10.io.spdk:cnode1
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_malloc_delete Malloc0

cd /opt/spdk

# 1) 创建两个 malloc bdev
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_malloc_create -b Malloc0 512 4096
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_malloc_create -b Malloc1 512 4096

# 2) 创建 RAID0（条带大小 64K）
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_raid_create \
    -n Raid0 -z 64 -r 0 -b "Malloc0 Malloc1"

# 3) 对 Raid0 设置 QoS（限制读写合计 IOPS 为 50000）
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_set_qos_limit \
    Raid0 --rw-ios-per-sec 50000

# 4) 映射到 NBD 验证
sudo modprobe nbd nbds_max=16
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock nbd_start_disk Raid0 /dev/nbd0

# 5) fio 验证 - 观察 IOPS 是否被限制在 50000 左右
sudo fio --name=raid0-qos \
    --filename=/dev/nbd0 \
    --ioengine=libaio \
    --direct=1 \
    --rw=randread \
    --bs=4k \
    --iodepth=64 \
    --numjobs=1 \
    --time_based \
    --runtime=10 \
    --group_reporting

# 6) 清理
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock nbd_stop_disk /dev/nbd0
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_raid_delete Raid0
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_malloc_delete Malloc0
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_malloc_delete Malloc1

cd /opt/spdk
sudo build/bin/nvmf_tgt -m 0x3 -s 4096 -r /var/tmp/spdk.sock &
sleep 2
RPC="sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock"

# 1) 创建 TCP transport
$RPC nvmf_create_transport -t TCP -u 8192 -m 8 -c 4096

# 2) 接管 NVMe 控制器（-a 是 PCI 地址，用 lspci 确认）
$RPC bdev_nvme_attach_controller -b Nvme0 -t pcie -a 0000:5e:00.0

# 3) 确认 namespace bdev 出现，通常是 Nvme0n1
$RPC bdev_get_bdevs | jq -r '.[].name'

# 在计算节点上取 host NQN，把输出的字符串记下来
nvme show-hostnqn
# 输出示例：nqn.2014-08.org.nvmexpress:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx

# 在存储节点上
RPC="sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock"
HOSTNQN="nqn.2014-08.org.nvmexpress:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
SUB=nqn.2025-10.io.example:storage0
$RPC nvmf_create_subsystem "$SUB" -s SPDK00000000000001
$RPC nvmf_subsystem_add_host "$SUB" "$HOSTNQN"
$RPC nvmf_subsystem_add_ns "$SUB" Nvme0n1
$RPC nvmf_subsystem_add_listener "$SUB" -t tcp -a 10.0.0.10 -s 4420

# bdev 层 IO 统计（最常用）
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_get_iostat | jq

# 输出结构（关键字段）：
# {
#   "tick_rate": 2200000000,          <- CPU TSC 频率
#   "ticks": 123456789,               <- 当前 tick
#   "bdevs": [{
#     "name": "Malloc0",
#     "bytes_read": 1048576,
#     "bytes_written": 2097152,
#     "num_read_ops": 256,
#     "num_write_ops": 512,
#     "read_latency_ticks": 12345,    <- 累计延迟 ticks，除以 num_read_ops 再换算
#     "write_latency_ticks": 23456,
#     ...
#   }]
# }

# NVMe-oF subsystem 连接信息
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock nvmf_get_stats | jq

# Reactor 线程负载
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock framework_get_reactors | jq

sudo cat > /usr/local/bin/spdk_exporter.sh <<'SCRIPT'
#!/usr/bin/env bash
# SPDK Prometheus Exporter - textfile collector
# 每次执行生成 .prom 文件供 node_exporter 采集

set -euo pipefail

SOCK="/var/tmp/spdk.sock"
RPC="/opt/spdk/scripts/rpc.py -s $SOCK"
OUT="/var/lib/node_exporter/textfile_collector/spdk.prom"
TMP=$(mktemp)

# 检查 SPDK 进程是否存活
if ! $RPC framework_get_reactors >/dev/null 2>&1; then
    echo '# SPDK process not running' > "$TMP"
    echo 'spdk_up 0' >> "$TMP"
    mv "$TMP" "$OUT"
    exit 0
fi

echo 'spdk_up 1' > "$TMP"

# bdev IO 统计
IOSTAT=$($RPC bdev_get_iostat 2>/dev/null || echo '{}')
TICK_RATE=$(echo "$IOSTAT" | jq -r '.tick_rate // 1')

echo "$IOSTAT" | jq -r --argjson tr "$TICK_RATE" '
    .bdevs[]? |
    "spdk_bdev_bytes_read{bdev=\"\(.name)\"} \(.bytes_read // 0)",
    "spdk_bdev_bytes_written{bdev=\"\(.name)\"} \(.bytes_written // 0)",
    "spdk_bdev_read_ops_total{bdev=\"\(.name)\"} \(.num_read_ops // 0)",
    "spdk_bdev_write_ops_total{bdev=\"\(.name)\"} \(.num_write_ops // 0)",
    "spdk_bdev_read_latency_us{bdev=\"\(.name)\"} \((if .num_read_ops > 0 then (.read_latency_ticks / .num_read_ops / $tr * 1000000) else 0 end))",
    "spdk_bdev_write_latency_us{bdev=\"\(.name)\"} \((if .num_write_ops > 0 then (.write_latency_ticks / .num_write_ops / $tr * 1000000) else 0 end))"
' >> "$TMP"

# Reactor 负载统计
$RPC framework_get_reactors 2>/dev/null | jq -r '
    .reactors[]? |
    "spdk_reactor_busy_ticks{core=\"\(.lcore)\"} \(.busy // 0)",
    "spdk_reactor_idle_ticks{core=\"\(.lcore)\"} \(.idle // 0)"
' >> "$TMP"

mv "$TMP" "$OUT"
chmod 0644 "$OUT"
SCRIPT

sudo chmod +x /usr/local/bin/spdk_exporter.sh
sudo mkdir -p /var/lib/node_exporter/textfile_collector

# 创建 service 单元
sudo cat > /etc/systemd/system/spdk-exporter.service <<'EOF'
[Unit]
Description=SPDK Prometheus Metrics Exporter

[Service]
Type=oneshot
ExecStart=/usr/local/bin/spdk_exporter.sh
User=root
EOF

# 创建 timer 单元（每 15 秒采集一次）
sudo cat > /etc/systemd/system/spdk-exporter.timer <<'EOF'
[Unit]
Description=Run SPDK exporter every 15 seconds

[Timer]
OnBootSec=30
OnUnitActiveSec=15

[Install]
WantedBy=timers.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable --now spdk-exporter.timer

# 验证
sudo systemctl list-timers | grep spdk
cat /var/lib/node_exporter/textfile_collector/spdk.prom

cd /opt/spdk
git fetch --tags

# 查看可用版本
git tag -l 'v25.*' | sort -V

# 以 v25.09 为例，换成你要升级的 tag
# 在不同目录编译新版本（不影响正在运行的旧版本）
sudo cp -a /opt/spdk /opt/spdk-v25.09
cd /opt/spdk-v25.09
git checkout v25.09
git submodule update --init --recursive
./configure --with-vhost --with-virtio --with-nvme-cuse
make -j$(nproc)

# 验证新版本编译成功
build/bin/spdk_tgt --version

# 把 systemd 指向新目录，但先不重启，等维护窗口再切
sudo sed -i 's|/opt/spdk/|/opt/spdk-v25.09/|g' /etc/systemd/system/spdk-tgt.service
sudo systemctl daemon-reload

# 导出当前版本的 RPC 方法列表
cd /opt/spdk
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock rpc_get_methods | jq -r '.[]' | sort > /tmp/rpc_methods_old.txt

# 编译新版本后启动临时实例，导出其方法列表
cd /opt/spdk-v25.09
sudo build/bin/spdk_tgt -m 0x4 -s 512 -r /var/tmp/spdk_test.sock &
sleep 2
sudo python3 scripts/rpc.py -s /var/tmp/spdk_test.sock rpc_get_methods | jq -r '.[]' | sort > /tmp/rpc_methods_new.txt
sudo pkill -f "spdk_tgt.*spdk_test.sock"

# 对比差异
diff /tmp/rpc_methods_old.txt /tmp/rpc_methods_new.txt

# 1) 确认 NVMe 设备的 NUMA 节点
cat /sys/bus/pci/devices/0000:5e:00.0/numa_node
# 输出示例: 1

# 2) 确认 NUMA 节点对应的 CPU 核心
lscpu | grep -i numa
# NUMA node0 CPU(s): 0-15,32-47
# NUMA node1 CPU(s): 16-31,48-63

# 3) 选择 NUMA node 1 的核心用于 SPDK（如核心 16-19）
# 核心掩码计算: cores 16,17,18,19 -> 0xF0000
sudo build/bin/spdk_tgt -m 0xF0000 -s 4096 -r /var/tmp/spdk.sock

# 4) 确保 hugepage 分配在 node 1
echo 2048 | sudo tee /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages

# TCP Transport 关键参数
sudo python3 scripts/rpc.py -s /var/tmp/spdk.sock nvmf_create_transport \
    -t TCP \
    -u 8192 \
    -m 8 \
    -c 4096 \
    -n 4096 \
    -q 128

# 参数说明：
# -u (io_unit_size)     : 单次 I/O 传输单元大小，建议 8192-131072
# -m (max_io_qpairs_per_ctrlr) : 每个控制器最大 I/O 队列对数
# -c (in_capsule_data_size)    : capsule 内嵌数据大小，小 I/O 场景可增大以减少 RTT
# -n (num_shared_buffers)      : 共享缓冲区数量，高并发场景需增大
# -q (max-queue-depth) : 每个队列最大未完成 I/O 数

cat <<'EOF' | sudo tee /etc/sysctl.d/61-spdk-network.conf
# TCP buffer sizes for NVMe-oF/TCP
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 87380 16777216

# Increase connection backlog
net.core.somaxconn = 4096
net.core.netdev_max_backlog = 8192

# Disable TCP slow start after idle (对 NVMe-oF 重要)
net.ipv4.tcp_slow_start_after_idle = 0
EOF

sudo sysctl -p /etc/sysctl.d/61-spdk-network.conf

# 1) JSON-RPC socket 权限限制（仅 root 和 spdk 组可访问）
sudo groupadd -r spdk 2>/dev/null || true
sudo chgrp spdk /var/tmp/spdk.sock
sudo chmod 0660 /var/tmp/spdk.sock

# 2) 如果要以非 root 用户运行，在 service 里配 User= 和所需 capabilities：
# CapabilityBoundingSet=CAP_SYS_ADMIN CAP_IPC_LOCK CAP_NET_ADMIN CAP_SYS_RAWIO
# AmbientCapabilities=CAP_SYS_ADMIN CAP_IPC_LOCK CAP_NET_ADMIN CAP_SYS_RAWIO
# 实际需要的 capability 随版本和驱动变化，先在测试环境确认。

# 3) 启用 NVMe-oF/TLS（PSK）
# PSK 是预先共享的密钥，不需要证书：
# python3 scripts/rpc.py nvmf_subsystem_add_host <subnqn> <hostnqn> --psk /etc/spdk/psk.txt
# python3 scripts/rpc.py nvmf_subsystem_add_listener <subnqn> -t tcp -a <ip> -s 4420 --secure-channel

#!/usr/bin/env bash
# /opt/spdk/conf/graceful-shutdown.sh
# 适用于本文的 malloc + NBD + NVMe-oF 拓扑。真实环境的 bdev 类型更多，删除步骤要按类型替换。

set -euo pipefail
RPC="/opt/spdk/scripts/rpc.py -s /var/tmp/spdk.sock"

echo "Step 1: 移除所有 NVMe-oF listener..."
$RPC nvmf_get_subsystems | jq -r '
    .[] | select(.nqn != "nqn.2014-08.org.nvmexpress.discovery") |
    .nqn as $nqn | .listeners[]? |
    "\($nqn) \(.trtype) \(.traddr) \(.trsvcid)"
' | while read -r sub trtype traddr trsvcid; do
    echo "  $sub $trtype $traddr $trsvcid"
    $RPC nvmf_subsystem_remove_listener "$sub" -t "$trtype" -a "$traddr" -s "$trsvcid" || true
done

echo "Step 2: 等待 initiator 断开或 I/O 排空..."
sleep 15

echo "Step 3: 删除 subsystem..."
$RPC nvmf_get_subsystems | jq -r '.[].nqn | select(. != "nqn.2014-08.org.nvmexpress.discovery")' |
while read -r sub; do
    $RPC nvmf_delete_subsystem "$sub"
done

echo "Step 4: 停止 NBD 映射..."
$RPC nbd_get_disks | jq -r '.[].nbd_device' 2>/dev/null |
while read -r nbd; do
    $RPC nbd_stop_disk "$nbd"
done

echo "Step 5: 删除 bdev（按实际类型替换命令）..."
$RPC bdev_malloc_delete Malloc0 2>/dev/null || true
# 真实 NVMe：$RPC bdev_nvme_detach_controller Nvme0
# RAID：$RPC bdev_raid_delete Raid0

echo "Step 6: 终止进程..."
$RPC spdk_kill_instance SIGTERM

#!/usr/bin/env bash
# spdk-quickstart.sh - Debian 12 SPDK 最小可复现验证
# 用法: sudo bash spdk-quickstart.sh

set -euo pipefail

echo "=== [1/7] Install dependencies ==="
apt update
apt install -y git build-essential meson ninja-build pkg-config \
    python3 python3-venv python3-pip \
    python3-pyelftools libnuma-dev libssl-dev libaio-dev libfuse3-dev \
    uuid-dev libncurses-dev libcunit1-dev autoconf automake libtool \
    jq curl fio nasm

echo "=== [2/7] Clone and build SPDK v24.09 ==="
cd /opt
[ -d spdk ] || git clone https://github.com/spdk/spdk.git
cd spdk
git checkout v24.09
git submodule update --init --recursive
./configure
make -j$(nproc)

echo "=== [3/7] Configure hugepages ==="
sysctl -w vm.nr_hugepages=1024
if [ "$(awk '/HugePages_Total/{print $2}' /proc/meminfo)" -lt 1024 ]; then
    # 内存碎片或 page cache 占得多时，内核可能只分配一部分
    sync
    echo 3 > /proc/sys/vm/drop_caches
    sysctl -w vm.nr_hugepages=1024
fi
mkdir -p /dev/hugepages
mount -t hugetlbfs nodev /dev/hugepages 2>/dev/null || true
grep HugePages_ /proc/meminfo
if [ "$(awk '/HugePages_Total/{print $2}' /proc/meminfo)" -lt 1024 ]; then
    echo "hugepage 分配不足，需要 1024 页（2GB）" >&2
    exit 1
fi

echo "=== [4/7] Start spdk_tgt ==="
# -u 跳过 PCI，纯 malloc 验证不需要 VFIO
build/bin/spdk_tgt -m 0x1 -s 1024 -u -r /var/tmp/spdk.sock &
SPDK_PID=$!
sleep 2
echo "spdk_tgt PID: $SPDK_PID"

echo "=== [5/7] Create malloc bdev and NBD mapping ==="
python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_malloc_create -b Malloc0 512 4096
modprobe nbd nbds_max=16
python3 scripts/rpc.py -s /var/tmp/spdk.sock nbd_start_disk Malloc0 /dev/nbd0
sleep 1
lsblk | grep nbd0

echo "=== [6/7] Run fio benchmark ==="
fio --name=spdk-verify \
    --filename=/dev/nbd0 \
    --ioengine=libaio \
    --direct=1 \
    --rw=randrw \
    --rwmixread=70 \
    --bs=4k \
    --iodepth=32 \
    --numjobs=1 \
    --time_based \
    --runtime=10 \
    --group_reporting

echo "=== [7/7] Cleanup ==="
python3 scripts/rpc.py -s /var/tmp/spdk.sock nbd_stop_disk /dev/nbd0
python3 scripts/rpc.py -s /var/tmp/spdk.sock bdev_malloc_delete Malloc0
kill $SPDK_PID 2>/dev/null; wait $SPDK_PID 2>/dev/null || true

echo "=== SPDK quickstart verification PASSED ==="