feat: support duplicate device IDs in dracut

internal/configs/config_bootloaders.go

@@ -51,7 +51,7 @@ func getBootloader(config *Config) {
 // This function adds the default kernel arguments we want to the config/cmdline file
 // This gives us a file we can read all the kernel arguments this system needs
 // in case of an unknown bootloader
-func Set_Cmdline(gpu_IDs []string) {
+func Set_Cmdline(gpu_IDs []string, includeDeviceIdsForVfio bool) {
 	// Get the system info
 	cpuinfo := cpuid.CPU
 
@@ -69,9 +69,11 @@ func Set_Cmdline(gpu_IDs []string) {
 		fileio.AppendContent(" intel_iommu=on", config.Path.CMDLINE)
 	}
 
+	if includeDeviceIdsForVfio {
 		// Add the GPU ids for vfio to the kernel arguments
 		fileio.AppendContent(fmt.Sprintf(" vfio_pci.ids=%s", strings.Join(gpu_IDs, ",")), config.Path.CMDLINE)
 	}
+}
 
 // Set_KernelStub configures systemd-boot using kernelstub.
 func Set_KernelStub(isRoot bool) {

internal/configs/config_dracut.go

@@ -3,16 +3,17 @@ package configs
 import (
 	"fmt"
 	"os"
+	"path"
+	"regexp"
 	"strings"
 
+	"github.com/HikariKnight/quickpassthrough/internal/common"
 	"github.com/HikariKnight/quickpassthrough/internal/logger"
 	"github.com/HikariKnight/quickpassthrough/pkg/fileio"
 )
 
 // Set_Dracut writes a dracut configuration file for `/etc/dracut.conf.d/`.
-func Set_Dracut() {
-	config := GetConfig()
-
+func Set_Dracut(config *Config) {
 	// Set the dracut config file
 	dracutConf := fmt.Sprintf("%s/vfio.conf", config.Path.DRACUT)
 
@@ -38,4 +39,61 @@ func Set_Dracut() {
 
 	// Make a backup of dracutConf if there is one there
 	backupFile(strings.Replace(dracutConf, "config", "", 1))
+
+	if config.HasDuplicateDeviceIds {
+		setDracutEarlyBinds(config)
+	}
+}
+
+func setDracutEarlyBinds(config *Config) {
+	err := os.MkdirAll(config.Path.DRACUTMODULE, os.ModePerm)
+	common.ErrorCheck(err, "Error, could not create dracut module config directory")
+	confToSystemPathRe := regexp.MustCompile(`^config`)
+
+	earlyBindScriptConfigPath := path.Join(config.Path.DRACUTMODULE, "early-vfio-bind.sh")
+	earlyBindScriptSysPath := confToSystemPathRe.ReplaceAllString(earlyBindScriptConfigPath, "")
+	config.EarlyBindFilePaths[earlyBindScriptConfigPath] = earlyBindScriptSysPath
+	if exists, _ := fileio.FileExist(earlyBindScriptConfigPath); exists {
+		_ = os.Remove(earlyBindScriptConfigPath)
+	}
+
+	logger.Printf("Writing to early bind script to %s", earlyBindScriptConfigPath)
+	vfioBindScript := fmt.Sprintf(`#!/bin/bash
+DEVS="%s"
+
+for DEV in $DEVS; do
+	echo "vfio-pci" > /sys/bus/pci/devices/$DEV/driver_override
+done
+
+# Load the vfio-pci module
+modprobe -i vfio-pci`, strings.Join(config.Gpu_Addresses, " "))
+
+	fileio.AppendContent(vfioBindScript, earlyBindScriptConfigPath)
+	err = os.Chmod(earlyBindScriptConfigPath, 0755)
+	common.ErrorCheck(err, fmt.Sprintf("Error, could not chmod %s", earlyBindScriptConfigPath))
+
+	dracutModuleConfigPath := path.Join(config.Path.DRACUTMODULE, "module-setup.sh")
+	dracutModuleSysPath := confToSystemPathRe.ReplaceAllString(dracutModuleConfigPath, "")
+	config.EarlyBindFilePaths[dracutModuleConfigPath] = dracutModuleSysPath
+	if exists, _ := fileio.FileExist(dracutModuleConfigPath); exists {
+		_ = os.Remove(dracutModuleConfigPath)
+	}
+
+	logger.Printf("Writing to dracut early bind config to %s", dracutModuleConfigPath)
+	dracutConfig := fmt.Sprintf(`#!/bin/bash
+check() {
+	return 0
+}
+
+depends() {
+	return 0
+}
+
+install() {
+	inst_hook pre-trigger 90 "$moddir/%s"
+}`, path.Base(earlyBindScriptSysPath))
+
+	fileio.AppendContent(dracutConfig, dracutModuleConfigPath)
+	err = os.Chmod(dracutModuleConfigPath, 0755)
+	common.ErrorCheck(err, fmt.Sprintf("Error, could not chmod %s", dracutModuleConfigPath))
 }

internal/configs/configs.go

@@ -24,6 +24,7 @@ type Path struct {
 	DEFAULT      string
 	QEMU         string
 	DRACUT       string
+	DRACUTMODULE string
 	MKINITCPIO   string
 }
 
@@ -33,7 +34,10 @@ type Config struct {
 	Path                  *Path
 	Gpu_Group             string
 	Gpu_IDs               []string
+	Gpu_Addresses         []string
+	EarlyBindFilePaths    map[string]string
 	IsRoot                bool
+	HasDuplicateDeviceIds bool
 }
 
 // GetConfigPaths retrieves the path to all the config files.
@@ -46,6 +50,7 @@ func GetConfigPaths() *Path {
 		DEFAULT:      "config/etc/default",
 		QEMU:         "config/qemu",
 		DRACUT:       "config/etc/dracut.conf.d",
+		DRACUTMODULE: "config/usr/lib/dracut/modules.d/90early-vfio-bind",
 		MKINITCPIO:   "config/etc/mkinitcpio.conf",
 	}
 
@@ -60,6 +65,9 @@ func GetConfig() *Config {
 		Path:                  GetConfigPaths(),
 		Gpu_Group:             "",
 		Gpu_IDs:               []string{},
+		Gpu_Addresses:         []string{},
+		EarlyBindFilePaths:    map[string]string{},
+		HasDuplicateDeviceIds: false,
 	}
 
 	// Detect the bootloader we are using
@@ -78,6 +86,7 @@ func InitConfigs() {
 		config.Path.INITRAMFS,
 		config.Path.DEFAULT,
 		config.Path.DRACUT,
+		config.Path.DRACUTMODULE,
 	}
 
 	// Remove old config

internal/pages/02_select_gpu.go

@@ -3,11 +3,13 @@ package pages
 import (
 	"fmt"
 	"os"
+	"regexp"
 
 	"github.com/gookit/color"
 
 	"github.com/HikariKnight/quickpassthrough/internal/common"
 	"github.com/HikariKnight/quickpassthrough/internal/configs"
+	"github.com/HikariKnight/quickpassthrough/internal/logger"
 	"github.com/HikariKnight/quickpassthrough/internal/lsiommu"
 	"github.com/HikariKnight/quickpassthrough/pkg/command"
 	"github.com/HikariKnight/quickpassthrough/pkg/fileio"
@@ -94,25 +96,21 @@ func viewGPU(config *configs.Config, ext ...int) {
 		// Get the device ids for the selected gpu using ls-iommu
 		config.Gpu_IDs = lsiommu.GetIOMMU("-g", mode, "-i", config.Gpu_Group, "--id")
 
-		// If the kernel_args file already exists
-		if exists, _ := fileio.FileExist(config.Path.CMDLINE); exists {
-			// Delete it as we will have to make a new one anyway
-			err := os.Remove(config.Path.CMDLINE)
-			common.ErrorCheck(err, fmt.Sprintf("Could not remove %s", config.Path.CMDLINE))
-		}
-
-		// Write initial kernel_arg file
-		configs.Set_Cmdline(config.Gpu_IDs)
-
-		// Go to the vbios dumper page
-		genVBIOS_dumper(config)
-
 	case "manual":
 		config.Gpu_IDs = menu.ManualInput(
 			"Please manually enter the vendorID:deviceID for every device to use except PCI Express Switches\n"+
 				"NOTE: All devices sharing the same IOMMU group will still get pulled into the VM!",
 			"xxxx:yyyy,xxxx:yyyy,xxxx:yyyy",
 		)
+	}
+
+	logger.Printf("Checking for duplicate device Ids")
+	hasDuplicateDeviceIds := detectDuplicateDeviceIds(config.Gpu_Group, config.Gpu_IDs)
+
+	if hasDuplicateDeviceIds {
+		config.HasDuplicateDeviceIds = true
+		config.Gpu_Addresses = lsiommu.GetIOMMU("-g", mode, "-i", config.Gpu_Group, "--pciaddr")
+	}
 
 	// If the kernel_args file already exists
 	if exists, _ := fileio.FileExist(config.Path.CMDLINE); exists {
@@ -122,9 +120,43 @@ func viewGPU(config *configs.Config, ext ...int) {
 	}
 
 	// Write initial kernel_arg file
-		configs.Set_Cmdline(config.Gpu_IDs)
+	configs.Set_Cmdline(config.Gpu_IDs, !config.HasDuplicateDeviceIds)
 
 	// Go to the vbios dumper page
 	genVBIOS_dumper(config)
 }
+
+func detectDuplicateDeviceIds(selectedGpuGroup string, selectedDeviceIds []string) bool {
+	// TODO: this would be made much simpler if ls-iommu allowed using the --id flag without
+	// the "-i" flag.
+	gpus := lsiommu.GetIOMMU("-g", "-F", "vendor:,prod_name,optional_revision:,device_id")
+	iommu_group_regex := regexp.MustCompile(`(\d{1,3})`)
+	iommuGroups := []string{}
+	for _, gpu := range gpus {
+		iommuGroup := iommu_group_regex.FindString(gpu)
+		iommuGroups = append(iommuGroups, iommuGroup)
+	}
+
+	allDeviceIds := []string{}
+	for _, group := range iommuGroups {
+		if group == selectedGpuGroup {
+			continue
+		}
+
+		deviceIds := lsiommu.GetIOMMU("-g", "-r", "-i", group, "--id")
+		for _, deviceId := range deviceIds {
+			allDeviceIds = append(allDeviceIds, deviceId)
+		}
+	}
+
+	for _, deviceId := range allDeviceIds {
+		for _, selectedDeviceId := range selectedDeviceIds {
+			if deviceId == selectedDeviceId {
+				logger.Printf("Found duplicate device id: %s", deviceId)
+				return true
+			}
+		}
+	}
+
+	return false
 }

internal/pages/06_finalize.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"os/user"
+	"strings"
 	"syscall"
 
 	"github.com/gookit/color"
@@ -29,7 +30,7 @@ func prepModules(config *configs.Config) {
 	// If we have a folder for dracut
 	if exists, _ := fileio.FileExist(config.Path.DRACUT); exists {
 		// Configure dracut
-		configs.Set_Dracut()
+		configs.Set_Dracut(config)
 	}
 
 	// If we have a mkinitcpio.conf file
@@ -209,6 +210,17 @@ func installPassthrough(config *configs.Config) {
 		// Copy dracut config to /etc/dracut.conf.d/vfio
 		configs.CopyToSystem(config.IsRoot, dracutFile, "/etc/dracut.conf.d/vfio")
 
+		if config.HasDuplicateDeviceIds {
+			moduleSysPath := strings.Replace(config.Path.DRACUTMODULE, "config", "", 1)
+			if err := command.ExecAndLogSudo(config.IsRoot, false, "mkdir", "-p", moduleSysPath); err != nil {
+				log.Fatalf("Failed to create dracut module directory: %s", err)
+			}
+
+			for configPath, sysPath := range config.EarlyBindFilePaths {
+				configs.CopyToSystem(config.IsRoot, configPath, sysPath)
+			}
+		}
+
 		// Get systeminfo
 		sysinfo := uname.New()