AmesianX · August 27, 2024 16:03 · Nov 27, 2022 · Nov 27, 2022 · Nov 27, 2022 · Nov 27, 2022
diff --git a/pve-trick-sriov-vf-configuration b/pve-trick-sriov-vf-configuration
@@ -8,7 +8,6 @@ https://zhuanlan.zhihu.com/p/91197211
 https://zhuanlan.zhihu.com/p/356437308
 https://zhiliao.h3c.com/Theme/details/24770
 https://blog.csdn.net/Jackykxy/article/details/120585563
-https://support.mellanox.com/s/article/howto-configure-sr-iov-for-connectx-3-with-kvm--infiniband-x
 
 ---
 Update grub to enable SR-IOV

diff --git a/pve-trick-sriov-vf-configuration b/pve-trick-sriov-vf-configuration
@@ -8,6 +8,7 @@ https://zhuanlan.zhihu.com/p/91197211
 https://zhuanlan.zhihu.com/p/356437308
 https://zhiliao.h3c.com/Theme/details/24770
 https://blog.csdn.net/Jackykxy/article/details/120585563
+https://support.mellanox.com/s/article/howto-configure-sr-iov-for-connectx-3-with-kvm--infiniband-x
 
 ---
 Update grub to enable SR-IOV

diff --git a/pve-trick-sriov-vf-configuration b/pve-trick-sriov-vf-configuration
@@ -102,4 +102,37 @@ reboot
 ---
 Inspect system logs
 
-dmesg | grep enp*s*f*
+dmesg | grep enp*s*f*
+
+---
+/etc/netplan example
+
+# /etc/netplan/main.yaml
+network:
+  ethernets:
+    ens16:
+      macaddress: 24:8a:07:b7:a8:08
+      addresses:
+      - 10.178.0.56/24
+      gateway4: 10.178.0.5
+      dhcp: false
+      dhcp6: false
+      link-local: []
+      nameservers:
+        addresses:
+        - 10.178.0.5
+
+  vlans:
+    vlan10:
+      dhcp4: true
+      dhcp4-overrides:
+          route-metric: 200
+      id: 10
+      link: eno1
+      macaddress: 00:00:00:00:00:10
+
+    vlan20:
+      dhcp4: true
+      dhcp4-overrides:
+          route-metric: 500
+
diff --git a/pve-trick-sriov-vf-configuration b/pve-trick-sriov-vf-configuration
@@ -65,6 +65,9 @@ Description=Script to enable SR-IOV on boot
 
 [Service]
 Type=simple
+# set PF UP
+ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0 up'
+ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f1 up'
 # start SR-IOV
 ExecStartPre=/usr/bin/bash -c '/usr/bin/echo 8 > /sys/class/net/enp67s0f0/device/sriov_numvfs'
 # ExecStartPre=/usr/bin/bash -c '/usr/bin/echo 8 > /sys/class/net/enp67s0f1/device/sriov_numvfs'
@@ -76,10 +79,7 @@ ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v3 addr 24:8a:07:b7:a8:04'
 ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v4 addr 24:8a:07:b7:a8:05'
 ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v5 addr 24:8a:07:b7:a8:06'
 ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v6 addr 24:8a:07:b7:a8:07'
-ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v6 addr 24:8a:07:b7:a8:08'
-# set PF UP
-ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0 up'
-ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f1 up'
+ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v7 addr 24:8a:07:b7:a8:08'
 # set VF UP
 ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v0 up'
 ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v1 up'
@@ -97,6 +97,7 @@ WantedBy=multi-user.target
 systemctl daemon-reload
 systemctl enable --now sriov.service
 reboot
+reboot
 
 ---
 Inspect system logs

diff --git a/pve-trick-sriov-vf-configuration b/pve-trick-sriov-vf-configuration
@@ -13,7 +13,7 @@ https://blog.csdn.net/Jackykxy/article/details/120585563
 Update grub to enable SR-IOV
 
 # update the following line in /etc/default/grub
-# /etc/default/grub
+vim /etc/default/grub
 Intel：
 GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on pci=realloc iommu=pt pci=assign-busses pcie_acs_override=downstream,multifunction nofb textonly nomodeset initcall_blacklist=sysfb_init"
 

diff --git a/pve-trick-sriov-vf-configuration b/pve-trick-sriov-vf-configuration
@@ -0,0 +1,104 @@
+# The following is a step-by-step example for enabling SR-IOV VF on Promxox 7+ with BCM57810 10G NIC
+
+---
+References
+
+https://forum.proxmox.com/threads/enabling-sr-iov-for-intel-nic-x550-t2-on-proxmox-6.56677/
+https://zhuanlan.zhihu.com/p/91197211
+https://zhuanlan.zhihu.com/p/356437308
+https://zhiliao.h3c.com/Theme/details/24770
+https://blog.csdn.net/Jackykxy/article/details/120585563
+
+---
+Update grub to enable SR-IOV
+
+# update the following line in /etc/default/grub
+# /etc/default/grub
+Intel：
+GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on pci=realloc iommu=pt pci=assign-busses pcie_acs_override=downstream,multifunction nofb textonly nomodeset initcall_blacklist=sysfb_init"
+
+AMD：
+GRUB_CMDLINE_LINUX_DEFAULT="quiet amd_iommu=on pci=realloc iommu=pt pci=assign-busses pcie_acs_override=downstream,multifunction nofb textonly nomodeset initcall_blacklist=sysfb_init"
+
+update-grub
+
+# Local modules
+vim /etc/modules
+
+# add the following modules
+vfio
+vfio_iommu_type1
+vfio_pci
+vfio_virqfd
+
+# IOMMU interrupt remapping
+echo "options vfio_iommu_type1 allow_unsafe_interrupts=1" > /etc/modprobe.d/iommu_unsafe_interrupts.conf
+echo "options kvm ignore_msrs=1" > /etc/modprobe.d/kvm.conf
+
+# Update kernel
+update-initramfs -u
+
+# Reboot
+reboot
+
+---
+Inspect SR-IOV Capabilities
+
+# After enabling SR-IOV in BIOS, you should be able to see the SR-IOV capabilities
+lspci -s "xx:xx" -vvv
+
+---
+Instantiate VFs
+
+echo 8 > /sys/class/net/enp67s0f0/device/sriov_numvfs
+
+# verify result
+lspci | grep –i ethernet
+ip addr
+
+---
+Create SR-IOV daemon service
+
+# /etc/systemd/system/sriov.service
+[Unit]
+Description=Script to enable SR-IOV on boot
+
+[Service]
+Type=simple
+# start SR-IOV
+ExecStartPre=/usr/bin/bash -c '/usr/bin/echo 8 > /sys/class/net/enp67s0f0/device/sriov_numvfs'
+# ExecStartPre=/usr/bin/bash -c '/usr/bin/echo 8 > /sys/class/net/enp67s0f1/device/sriov_numvfs'
+# set VF MAC
+ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v0 addr 24:8a:07:b7:a8:01'
+ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v1 addr 24:8a:07:b7:a8:02'
+ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v2 addr 24:8a:07:b7:a8:03'
+ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v3 addr 24:8a:07:b7:a8:04'
+ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v4 addr 24:8a:07:b7:a8:05'
+ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v5 addr 24:8a:07:b7:a8:06'
+ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v6 addr 24:8a:07:b7:a8:07'
+ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v6 addr 24:8a:07:b7:a8:08'
+# set PF UP
+ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0 up'
+ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f1 up'
+# set VF UP
+ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v0 up'
+ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v1 up'
+ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v2 up'
+ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v3 up'
+ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v4 up'
+ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v5 up'
+ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v6 up'
+ExecStart=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v7 up'
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+
+systemctl daemon-reload
+systemctl enable --now sriov.service
+reboot
+
+---
+Inspect system logs
+
+dmesg | grep enp*s*f*