feat: support duplicate device IDs in dracut

internal/configs/config_bootloaders.go

@@ -51,7 +51,7 @@ func getBootloader(config *Config) {
 // This function adds the default kernel arguments we want to the config/cmdline file
 // This gives us a file we can read all the kernel arguments this system needs
 // in case of an unknown bootloader
-func Set_Cmdline(gpu_IDs []string) {
+func Set_Cmdline(gpu_IDs []string, includeDeviceIdsForVfio bool) {
 	// Get the system info
 	cpuinfo := cpuid.CPU
 
@@ -69,8 +69,10 @@ func Set_Cmdline(gpu_IDs []string) {
 		fileio.AppendContent(" intel_iommu=on", config.Path.CMDLINE)
 	}
 
-	// Add the GPU ids for vfio to the kernel arguments
+	if includeDeviceIdsForVfio {
-	fileio.AppendContent(fmt.Sprintf(" vfio_pci.ids=%s", strings.Join(gpu_IDs, ",")), config.Path.CMDLINE)
+		// Add the GPU ids for vfio to the kernel arguments
+		fileio.AppendContent(fmt.Sprintf(" vfio_pci.ids=%s", strings.Join(gpu_IDs, ",")), config.Path.CMDLINE)
+	}
 }
 
 // Set_KernelStub configures systemd-boot using kernelstub.

internal/configs/config_dracut.go

@@ -3,16 +3,17 @@ package configs
 import (
 	"fmt"
 	"os"
+	"path"
+	"regexp"
 	"strings"
 
+	"github.com/HikariKnight/quickpassthrough/internal/common"
 	"github.com/HikariKnight/quickpassthrough/internal/logger"
 	"github.com/HikariKnight/quickpassthrough/pkg/fileio"
 )
 
 // Set_Dracut writes a dracut configuration file for `/etc/dracut.conf.d/`.
-func Set_Dracut() {
+func Set_Dracut(config *Config) {
-	config := GetConfig()
-
 	// Set the dracut config file
 	dracutConf := fmt.Sprintf("%s/vfio.conf", config.Path.DRACUT)
 
@@ -38,4 +39,61 @@ func Set_Dracut() {
 
 	// Make a backup of dracutConf if there is one there
 	backupFile(strings.Replace(dracutConf, "config", "", 1))
+
+	if config.HasDuplicateDeviceIds {
+		setDracutEarlyBinds(config)
+	}
+}
+
+func setDracutEarlyBinds(config *Config) {
+	err := os.MkdirAll(config.Path.DRACUTMODULE, os.ModePerm)
+	common.ErrorCheck(err, "Error, could not create dracut module config directory")
+	confToSystemPathRe := regexp.MustCompile(`^config`)
+
+	earlyBindScriptConfigPath := path.Join(config.Path.DRACUTMODULE, "early-vfio-bind.sh")
+	earlyBindScriptSysPath := confToSystemPathRe.ReplaceAllString(earlyBindScriptConfigPath, "")
+	config.EarlyBindFilePaths[earlyBindScriptConfigPath] = earlyBindScriptSysPath
+	if exists, _ := fileio.FileExist(earlyBindScriptConfigPath); exists {
+		_ = os.Remove(earlyBindScriptConfigPath)
+	}
+
+	logger.Printf("Writing to early bind script to %s", earlyBindScriptConfigPath)
+	vfioBindScript := fmt.Sprintf(`#!/bin/bash
+DEVS="%s"
+
+for DEV in $DEVS; do
+	echo "vfio-pci" > /sys/bus/pci/devices/$DEV/driver_override
+done
+
+# Load the vfio-pci module
+modprobe -i vfio-pci`, strings.Join(config.Gpu_Addresses, " "))
+
+	fileio.AppendContent(vfioBindScript, earlyBindScriptConfigPath)
+	err = os.Chmod(earlyBindScriptConfigPath, 0755)
+	common.ErrorCheck(err, fmt.Sprintf("Error, could not chmod %s", earlyBindScriptConfigPath))
+
+	dracutModuleConfigPath := path.Join(config.Path.DRACUTMODULE, "module-setup.sh")
+	dracutModuleSysPath := confToSystemPathRe.ReplaceAllString(dracutModuleConfigPath, "")
+	config.EarlyBindFilePaths[dracutModuleConfigPath] = dracutModuleSysPath
+	if exists, _ := fileio.FileExist(dracutModuleConfigPath); exists {
+		_ = os.Remove(dracutModuleConfigPath)
+	}
+
+	logger.Printf("Writing to dracut early bind config to %s", dracutModuleConfigPath)
+	dracutConfig := fmt.Sprintf(`#!/bin/bash
+check() {
+	return 0
+}
+
+depends() {
+	return 0
+}
+
+install() {
+	inst_hook pre-trigger 90 "$moddir/%s"
+}`, path.Base(earlyBindScriptSysPath))
+
+	fileio.AppendContent(dracutConfig, dracutModuleConfigPath)
+	err = os.Chmod(dracutModuleConfigPath, 0755)
+	common.ErrorCheck(err, fmt.Sprintf("Error, could not chmod %s", dracutModuleConfigPath))
 }

internal/configs/configs.go

@@ -17,36 +17,41 @@ import (
 )
 
 type Path struct {
-	CMDLINE    string
+	CMDLINE      string
-	MODPROBE   string
+	MODPROBE     string
-	INITRAMFS  string
+	INITRAMFS    string
-	ETCMODULES string
+	ETCMODULES   string
-	DEFAULT    string
+	DEFAULT      string
-	QEMU       string
+	QEMU         string
-	DRACUT     string
+	DRACUT       string
-	MKINITCPIO string
+	DRACUTMODULE string
+	MKINITCPIO   string
 }
 
 type Config struct {
-	Bootloader string
+	Bootloader            string
-	Cpuvendor  string
+	Cpuvendor             string
-	Path       *Path
+	Path                  *Path
-	Gpu_Group  string
+	Gpu_Group             string
-	Gpu_IDs    []string
+	Gpu_IDs               []string
-	IsRoot     bool
+	Gpu_Addresses         []string
+	EarlyBindFilePaths    map[string]string
+	IsRoot                bool
+	HasDuplicateDeviceIds bool
 }
 
 // GetConfigPaths retrieves the path to all the config files.
 func GetConfigPaths() *Path {
 	Paths := &Path{
-		CMDLINE:    "config/kernel_args",
+		CMDLINE:      "config/kernel_args",
-		MODPROBE:   "config/etc/modprobe.d",
+		MODPROBE:     "config/etc/modprobe.d",
-		INITRAMFS:  "config/etc/initramfs-tools",
+		INITRAMFS:    "config/etc/initramfs-tools",
-		ETCMODULES: "config/etc/modules",
+		ETCMODULES:   "config/etc/modules",
-		DEFAULT:    "config/etc/default",
+		DEFAULT:      "config/etc/default",
-		QEMU:       "config/qemu",
+		QEMU:         "config/qemu",
-		DRACUT:     "config/etc/dracut.conf.d",
+		DRACUT:       "config/etc/dracut.conf.d",
-		MKINITCPIO: "config/etc/mkinitcpio.conf",
+		DRACUTMODULE: "config/usr/lib/dracut/modules.d/90early-vfio-bind",
+		MKINITCPIO:   "config/etc/mkinitcpio.conf",
 	}
 
 	return Paths
@@ -55,11 +60,14 @@ func GetConfigPaths() *Path {
 // GetConfig retrieves all the configs and returns the struct.
 func GetConfig() *Config {
 	config := &Config{
-		Bootloader: "unknown",
+		Bootloader:            "unknown",
-		Cpuvendor:  cpuid.CPU.VendorString,
+		Cpuvendor:             cpuid.CPU.VendorString,
-		Path:       GetConfigPaths(),
+		Path:                  GetConfigPaths(),
-		Gpu_Group:  "",
+		Gpu_Group:             "",
-		Gpu_IDs:    []string{},
+		Gpu_IDs:               []string{},
+		Gpu_Addresses:         []string{},
+		EarlyBindFilePaths:    map[string]string{},
+		HasDuplicateDeviceIds: false,
 	}
 
 	// Detect the bootloader we are using
@@ -78,6 +86,7 @@ func InitConfigs() {
 		config.Path.INITRAMFS,
 		config.Path.DEFAULT,
 		config.Path.DRACUT,
+		config.Path.DRACUTMODULE,
 	}
 
 	// Remove old config

internal/pages/02_select_gpu.go

@@ -3,11 +3,13 @@ package pages
 import (
 	"fmt"
 	"os"
+	"regexp"
 
 	"github.com/gookit/color"
 
 	"github.com/HikariKnight/quickpassthrough/internal/common"
 	"github.com/HikariKnight/quickpassthrough/internal/configs"
+	"github.com/HikariKnight/quickpassthrough/internal/logger"
 	"github.com/HikariKnight/quickpassthrough/internal/lsiommu"
 	"github.com/HikariKnight/quickpassthrough/pkg/command"
 	"github.com/HikariKnight/quickpassthrough/pkg/fileio"
@@ -94,37 +96,67 @@ func viewGPU(config *configs.Config, ext ...int) {
 		// Get the device ids for the selected gpu using ls-iommu
 		config.Gpu_IDs = lsiommu.GetIOMMU("-g", mode, "-i", config.Gpu_Group, "--id")
 
-		// If the kernel_args file already exists
-		if exists, _ := fileio.FileExist(config.Path.CMDLINE); exists {
-			// Delete it as we will have to make a new one anyway
-			err := os.Remove(config.Path.CMDLINE)
-			common.ErrorCheck(err, fmt.Sprintf("Could not remove %s", config.Path.CMDLINE))
-		}
-
-		// Write initial kernel_arg file
-		configs.Set_Cmdline(config.Gpu_IDs)
-
-		// Go to the vbios dumper page
-		genVBIOS_dumper(config)
-
 	case "manual":
 		config.Gpu_IDs = menu.ManualInput(
 			"Please manually enter the vendorID:deviceID for every device to use except PCI Express Switches\n"+
 				"NOTE: All devices sharing the same IOMMU group will still get pulled into the VM!",
 			"xxxx:yyyy,xxxx:yyyy,xxxx:yyyy",
 		)
+	}
 
-		// If the kernel_args file already exists
+	logger.Printf("Checking for duplicate device Ids")
-		if exists, _ := fileio.FileExist(config.Path.CMDLINE); exists {
+	hasDuplicateDeviceIds := detectDuplicateDeviceIds(config.Gpu_Group, config.Gpu_IDs)
-			// Delete it as we will have to make a new one anyway
+
-			err := os.Remove(config.Path.CMDLINE)
+	if hasDuplicateDeviceIds {
-			common.ErrorCheck(err, fmt.Sprintf("Could not remove %s", config.Path.CMDLINE))
+		config.HasDuplicateDeviceIds = true
+		config.Gpu_Addresses = lsiommu.GetIOMMU("-g", mode, "-i", config.Gpu_Group, "--pciaddr")
+	}
+
+	// If the kernel_args file already exists
+	if exists, _ := fileio.FileExist(config.Path.CMDLINE); exists {
+		// Delete it as we will have to make a new one anyway
+		err := os.Remove(config.Path.CMDLINE)
+		common.ErrorCheck(err, fmt.Sprintf("Could not remove %s", config.Path.CMDLINE))
+	}
+
+	// Write initial kernel_arg file
+	configs.Set_Cmdline(config.Gpu_IDs, !config.HasDuplicateDeviceIds)
+
+	// Go to the vbios dumper page
+	genVBIOS_dumper(config)
+}
+
+func detectDuplicateDeviceIds(selectedGpuGroup string, selectedDeviceIds []string) bool {
+	// TODO: this would be made much simpler if ls-iommu allowed using the --id flag without
+	// the "-i" flag.
+	gpus := lsiommu.GetIOMMU("-g", "-F", "vendor:,prod_name,optional_revision:,device_id")
+	iommu_group_regex := regexp.MustCompile(`(\d{1,3})`)
+	iommuGroups := []string{}
+	for _, gpu := range gpus {
+		iommuGroup := iommu_group_regex.FindString(gpu)
+		iommuGroups = append(iommuGroups, iommuGroup)
+	}
+
+	allDeviceIds := []string{}
+	for _, group := range iommuGroups {
+		if group == selectedGpuGroup {
+			continue
 		}
 
-		// Write initial kernel_arg file
+		deviceIds := lsiommu.GetIOMMU("-g", "-r", "-i", group, "--id")
-		configs.Set_Cmdline(config.Gpu_IDs)
+		for _, deviceId := range deviceIds {
-
+			allDeviceIds = append(allDeviceIds, deviceId)
-		// Go to the vbios dumper page
+		}
-		genVBIOS_dumper(config)
 	}
+
+	for _, deviceId := range allDeviceIds {
+		for _, selectedDeviceId := range selectedDeviceIds {
+			if deviceId == selectedDeviceId {
+				logger.Printf("Found duplicate device id: %s", deviceId)
+				return true
+			}
+		}
+	}
+
+	return false
 }

internal/pages/06_finalize.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"os/user"
+	"strings"
 	"syscall"
 
 	"github.com/gookit/color"
@@ -29,7 +30,7 @@ func prepModules(config *configs.Config) {
 	// If we have a folder for dracut
 	if exists, _ := fileio.FileExist(config.Path.DRACUT); exists {
 		// Configure dracut
-		configs.Set_Dracut()
+		configs.Set_Dracut(config)
 	}
 
 	// If we have a mkinitcpio.conf file
@@ -209,6 +210,17 @@ func installPassthrough(config *configs.Config) {
 		// Copy dracut config to /etc/dracut.conf.d/vfio
 		configs.CopyToSystem(config.IsRoot, dracutFile, "/etc/dracut.conf.d/vfio")
 
+		if config.HasDuplicateDeviceIds {
+			moduleSysPath := strings.Replace(config.Path.DRACUTMODULE, "config", "", 1)
+			if err := command.ExecAndLogSudo(config.IsRoot, false, "mkdir", "-p", moduleSysPath); err != nil {
+				log.Fatalf("Failed to create dracut module directory: %s", err)
+			}
+
+			for configPath, sysPath := range config.EarlyBindFilePaths {
+				configs.CopyToSystem(config.IsRoot, configPath, sysPath)
+			}
+		}
+
 		// Get systeminfo
 		sysinfo := uname.New()