PVE8 需要16.0+版本的vgpu驱动 低版本不支持6.x内核
主板 BIOS 配置需要开启 IOMMU / VT-d、Above 4G Decoding、SR-IOV,不同的主板选项所在位置可能不同,需要自行查找。
修改相关配置及参数
vim /etc/default/grub
#找到
GRUB_CMDLINE_LINUX_DEFAULT="quiet"
#intel cpu 改为:
GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt spectre_v2=off spectre_v1=off pcie_acs_override=downstream video=efifb:off"
#amd cpu改为:
GRUB_CMDLINE_LINUX_DEFAULT="quiet amd_iommu=on iommu=pt"
update-grub
echo vfio >> /etc/modules
echo vfio_iommu_type1 >> /etc/modules
echo vfio_pci >> /etc/modules
echo vfio_virqfd >> /etc/modules
#AMD显卡
echo "blacklist radeon" >> /etc/modprobe.d/blacklist.conf
echo "blacklist amdgpu" >> /etc/modprobe.d/blacklist.conf
#NVIDIA显卡
echo "blacklist nouveau" >> /etc/modprobe.d/blacklist.conf
echo "blacklist nvidia" >> /etc/modprobe.d/blacklist.conf
echo "blacklist nvidiafb" >> /etc/modprobe.d/blacklist.conf
#INTEL核显
echo "blacklist snd_hda_intel" >> /etc/modprobe.d/blacklist.conf
echo "blacklist snd_hda_codec_hdmi" >> /etc/modprobe.d/blacklist.conf
echo "blacklist i915" >> /etc/modprobe.d/blacklist.conf
echo "options kvm ignore_msrs=1" > /etc/modprobe.d/kvm.conf
update-initramfs -u -k all
安装包
apt update
apt install -y build-essential dkms mdevctl pve-headers-$(uname -r)
#进入驱动所在目录
#r如果安装了显卡驱动,先卸载
./NVIDIA-Linux-x86_64-535.54.06-vgpu-kvm.run --uninstall
sudo apt-get remove --purge nvidia-*
###卸载后安装
chmod +x NVIDIA-Linux-x86_64-535.54.06-vgpu-kvm.run
./NVIDIA-Linux-x86_64-535.54.06-vgpu-kvm.run
###Linux-6.8内核
./NVIDIA-Linux-x86_64-550.54.16-vgpu-kvm.run --apply-patch vGPU-Grid-17.1-550.54.16-linux-6.8.patch
###17.0版本以上Pascal及Maxwell的多数卡已经废弃支持需要指定-m=kernel参数
./NVIDIA-Linux-x86_64-550.54.10-vgpu-kvm-custom.run -m kernel
#重启pve
reboot
创建配置非专业卡
mkdir /etc/vgpu_unlock
touch /etc/vgpu_unlock/profile_override.toml
mkdir /etc/systemd/system/{nvidia-vgpud.service.d,nvidia-vgpu-mgr.service.d}
echo -e "[Service]\nEnvironment=LD_PRELOAD=/opt/vgpu_unlock-rs/target/release/libvgpu_unlock_rs.so" > /etc/systemd/system/nvidia-vgpud.service.d/vgpu_unlock.conf
echo -e "[Service]\nEnvironment=LD_PRELOAD=/opt/vgpu_unlock-rs/target/release/libvgpu_unlock_rs.so" > /etc/systemd/system/nvidia-vgpu-mgr.service.d/vgpu_unlock.conf
cat /etc/systemd/system/{nvidia-vgpud.service.d,nvidia-vgpu-mgr.service.d}/*
[Service]
Environment=LD_PRELOAD=/opt/vgpu_unlock-rs/target/release/libvgpu_unlock_rs.so
[Service]
Environment=LD_PRELOAD=/opt/vgpu_unlock-rs/target/release/libvgpu_unlock_rs.so
上传必须文件
随后通过ssh或者是其他方式上传libvgpu_unlock_rs.so
mkdir -p /opt/vgpu_unlock-rs/target/release
cd /opt/vgpu_unlock-rs/target/release
chmod +x NVIDIA-Linux-x86_64-525.105.14-vgpu-kvm-kernel-6.2.9-patched.run
安装驱动
./NVIDIA-Linux-x86_64-525.105.14-vgpu-kvm-kernel-6.2.9-patched.run
systemctl restart {nvidia-vgpud.service,nvidia-vgpu-mgr.service}
测试
nvidia-smi
关闭持久模式
nvidia-smi -pm 0 1为开启
mdevctl types 查看id
解锁帧数限制
/etc/vgpu_unlock/profile_override.toml
[profile.nvidia-260]
frl_enabled = 0
其他配置
[profile.nvidia-260]
num_displays = 1 //一个显示器
display_width = 1920
display_height = 1080
max_pixels = 2073600 //分辨率
cuda_enabled = 1 //开启cuda
frl_enabled = 0 //取消帧率限制
TDR或OpenGL性能异常
[profile.nvidia-260]
vgpu_type = "NVS"
编译解锁补丁
git clone https://github.com/mbilker/vgpu_unlock-rs.git
cd ~
curl --proto '=https' --tlsv1.2 -sSf https://rsproxy.cn/rustup-init.sh | sh
source $HOME/.cargo/env
cd vgpu_unlock-rs
cargo build --release
NVIDIA VGPU授权fastapi-dls
docker run -d -e DLS_URL=10.10.10.1 -e DLS_PORT=443 -p 443:443 makedie/fastapi-dls
于虚拟机内使用浏览器,访问刚刚创建的docker容器,https://10.10.10.1/-/client-token
授权查看
win
# <ls-hostname-or-ip>为你的授权服务IP端口
curl.exe --insecure -L -X GET https://<dls-hostname-or-ip>/-/client-token -o "C:\Program Files\NVIDIA Corporation\vGPU Licensing\ClientConfigToken\client_configuration_token_$($(Get-Date).tostring('dd-MM-yy-hh-mm-ss')).tok"
Restart-Service NVDisplay.ContainerLocalSystem #重启英伟达服务
nvidia-smi.exe -q | Select-String License #查看授权状态
Linux
curl --insecure -L -X GET https://<dls-hostname-or-ip>/-/client-token -o /etc/nvidia/ClientConfigToken/client_configuration_token_$(date '+%d-%m-%Y-%H-%M-%S').tok
service nvidia-gridd restart #重启英伟达服务
nvidia-smi -q | grep License #查看授权状态
驱动下载
https://foxi.buduanwang.vip/pan/
https://github.com/justin-himself/NVIDIA-VGPU-Driver-Archive
https://cloud.google.com/compute/docs/gpus/grid-drivers-table?hl=zh-cn#windows_drivers
Ubuntu 安装VGPU驱动
配置root账号密码
sudo passwd root
关闭防火墙和DNS
sudo systemctl disable --now ufw
systemctl stop systemd-resolved
systemctl disable systemd-resolved
更新
sudo apt update
sudo apt upgrade
apt install build-essential gcc-multilib dkms mdevctl
echo 'blacklist nouveau' | sudo tee -a /etc/modprobe.d/blacklist.conf
echo 'options nouveau modeset=0' | sudo tee -a /etc/modprobe.d/blacklist.conf
update-initramfs -k all -u
sudo ./NVIDIA-Linux-x86_64-535.161.07-grid.run -s
sudo ./NVIDIA-Linux-x86_64-535.161.07-grid.run --no-x-check --no-nouveau-check --no-opengl-files -s
报错 x server
--no-x-check
docker 使用GPU信息
官网:https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
&& curl -s -L https://mirrors.ustc.edu.cn/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
sed 's#deb https://nvidia.github.io#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://mirrors.ustc.edu.cn#g' | \
sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
sudo apt update && sudo apt install nvidia-container-toolkit
一键配置docker
nvidia-ctk runtime configure --runtime=docker
{
"registry-mirrors": [
"https://docker.mirrors.sjtug.sjtu.edu.cn",
"https://docker.nju.edu.cn"
],
"runtimes": {
"nvidia": {
"args": [],
"path": "nvidia-container-runtime"
}
}
}
/etc/docker/daemon.json
sudo systemctl enable docker
systemctl restart docker
sudo apt-mark hold PACKAGE_NAME #锁定软件版本。可以一次指定多个包
sudo apt-mark showhold #显示系统中被标记为锁定的软件包
sudo apt-mark unhold PACKAGE_NAME #解除对软件包的锁定。可以一次指定多个包。
docker run -d --gpus all --restart="always" -e TZ=Asia/Shanghai -p 61208-61209:61208-61209 --privileged -e GLANCES_OPT="-w" -v /var/run/docker.sock:/var/run/docker.sock:ro --pid host --net=host --name glances nicolargo/glances:ubuntu-3.4.0.5-full
修改Emby网页端播放视频默认为静音
/var/packages/EmbyServer/target/system/dashboard-ui/modules/htmlvideoplayer/plugin.js
搜索 <video ,找到截图里这一段
对其video标签添加 muted 属性,然后重启Emby,刷新浏览器缓存,即可实现播放视频时默认静音
注:仅针对PC端网页播放
Ubuntu docker 映射GPU
# 测试,如果能正常输出显卡信息则安装成功
docker run --rm --runtime=nvidia --gpus all nvidia/cuda:11.6.2-base-ubuntu20.04 nvidia-smi
docker run \
--network=bridge \
-p '8095:8096' \
-p '8920:8920' \
-e JELLYFIN_FFmpeg__probesize="10M" \
-e JELLYFIN_FFmpeg__analyzeduration="5M" \
-v /xiaoya/docker/jellyfin/transcodes:/config/transcodes \
-v /xiaoya/docker/jellyfin:/config \
-v /xiaoya/docker/jellyfin/cache:/cache \
-v /xiaoya:/media \
--add-host=xiaoya.host:10.10.10.1 \
-e TZ="Asia/Shanghai" \
--restart always \
--runtime=nvidia \
-e NVIDIA_VISIBLE_DEVICES=all \
-e NVIDIA_DRIVER_CAPABILITIES=all \
--gpus all \
--name jellyfin \
-d nyanmisaka/jellyfin:latest
jellyfin/jellyfin:latest #官方
docker run \
--network=bridge \
-p '8096:8096' \
-v /dev/shm:/config/transcodes \
-v /root/emby:/config \
-v /xiaoya:/media \
--add-host=xiaoya.host:10.10.10.1 \
-e TZ="Asia/Shanghai" \
--restart always \
--runtime=nvidia \
-e NVIDIA_VISIBLE_DEVICES=all \
-e NVIDIA_DRIVER_CAPABILITIES=all \
--gpus all \
--name emby \
-d amilys/embyserver
群辉vgpu
vgpuDaemon fix
nvidia-smi
docker run \
--network=bridge \
-p '8096:8096' \
-v /volume1/docker/emby:/config \
-v /volume1/nas/media:/media \
-e TZ="Asia/Shanghai" \
-e UID=1026 \
-e GID=100 \
-e GIDLIST=0 \
--restart always \
--runtime=nvidia \
-e NVIDIA_VISIBLE_DEVICES=all \
-e NVIDIA_DRIVER_CAPABILITIES=all \
-e UMASK=000 \
--name emby \
-d lovechen/embyserver:latest
amilys/embyserver #美化破解
docker run \
--network=bridge \
-p '8096:8096' \
-p '8920:8920' \
-v /volume1/docker/jellyfin:/config \
-v /volume1/video/:/media \
-e TZ="Asia/Shanghai" \
-e UID=1026 \
-e GID=100 \
-e GIDLIST=0 \
--restart always \
--runtime=nvidia \
-e NVIDIA_VISIBLE_DEVICES=all \
-e NVIDIA_DRIVER_CAPABILITIES=all \
-e UMASK=000 \
--name jellyfin \
-d jellyfin/jellyfin:latest