Skip to content

[vfio-manage] migrate to go-nvlib/pkg/nvpassthrough #192

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
cdesiniotis wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

[vfio-manage] migrate to go-nvlib/pkg/nvpassthrough #192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 17 additions & 10 deletions cmd/vfio-manage/bind.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"

"github.com/NVIDIA/k8s-driver-manager/internal/nvpassthrough"
"github.com/NVIDIA/go-nvlib/pkg/nvpassthrough"
)

type bindCommand struct {
Expand All @@ -38,7 +38,7 @@ type bindCommand struct {
type bindOptions struct {
all bool
deviceID string
hostRoot string
libModulesRoot string
bindNVSwitches bool
}

Expand Down Expand Up @@ -75,18 +75,23 @@ func (m bindCommand) build() *cli.Command {
Usage: "Specific device ID to bind (e.g., 0000:01:00.0)",
},
&cli.StringFlag{
Name: "host-root",
Destination: &m.options.hostRoot,
EnvVars: []string{"HOST_ROOT"},
Value: "/",
Usage: "Path to the host's root filesystem. This is used when loading the vfio-pci module.",
Name: "host-root",
EnvVars: []string{"HOST_ROOT"},
Usage: "DEPRECATED: the host root is no longer required to load the vfio-pci module, please use --lib-modules-root instead",
},
&cli.BoolFlag{
Name: "bind-nvswitches",
Destination: &m.options.bindNVSwitches,
EnvVars: []string{"BIND_NVSWITCHES"},
Usage: "Also bind NVSwitches to vfio-pci (default: false)",
},
&cli.StringFlag{
Name: "lib-modules-root",
Destination: &m.options.libModulesRoot,
EnvVars: []string{"LIB_MODULES_ROOT"},
Value: "/lib/modules",
Usage: "Path to the /lib/modules. This is used when loading the vfio-pci module.",
},
},
}

Expand All @@ -112,7 +117,9 @@ func (m bindCommand) run() error {

m.nvpassthrough = nvpassthrough.New(
nvpassthrough.WithLogger(m.logger),
nvpassthrough.WithHostRoot(m.options.hostRoot),
nvpassthrough.WithLibModulesRoot(m.options.libModulesRoot),
nvpassthrough.WithNvpciLib(m.nvpci),
nvpassthrough.WithLoadKernelModules(true),
)

if m.options.deviceID != "" {
Expand All @@ -138,7 +145,7 @@ func (m bindCommand) bindAll() error {

for _, dev := range devices {
m.logger.Infof("Binding device %s", dev.Address)
if err := m.nvpassthrough.BindToVFIODriver(dev); err != nil {
if err := m.nvpassthrough.BindToVFIODriver(dev.Address); err != nil {
m.logger.Warnf("Failed to bind device %s: %v", dev.Address, err)
}
}
Expand Down Expand Up @@ -169,7 +176,7 @@ func (m bindCommand) bindDevice() error {

m.logger.Infof("Binding device %s", device)

if err := m.nvpassthrough.BindToVFIODriver(nvdev); err != nil {
if err := m.nvpassthrough.BindToVFIODriver(device); err != nil {
return fmt.Errorf("failed to bind device %s to vfio driver: %w", device, err)
}

Expand Down
20 changes: 10 additions & 10 deletions cmd/vfio-manage/unbind.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,10 @@ package main
import (
"fmt"

"github.com/NVIDIA/go-nvlib/pkg/nvpassthrough"
"github.com/NVIDIA/go-nvlib/pkg/nvpci"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"

"github.com/NVIDIA/k8s-driver-manager/internal/nvpassthrough"
)

type unbindCommand struct {
Expand All @@ -43,13 +42,16 @@ type unbindOptions struct {

// newUnbindCommand constructs an unbind command with the specified logger
func newUnbindCommand(logger *logrus.Logger) *cli.Command {
nvpciLib := nvpci.New(
nvpci.WithLogger(logger),
)

c := unbindCommand{
logger: logger,
nvpci: nvpci.New(
nvpci.WithLogger(logger),
),
nvpci: nvpciLib,
nvpassthrough: nvpassthrough.New(
nvpassthrough.WithLogger(logger),
nvpassthrough.WithNvpciLib(nvpciLib),
),
}
return c.build()
Expand Down Expand Up @@ -127,7 +129,7 @@ func (m unbindCommand) unbindAll() error {

for _, dev := range devices {
m.logger.Infof("Unbinding device %s", dev.Address)
if err := m.nvpassthrough.UnbindFromDriver(dev); err != nil {
if err := m.nvpassthrough.Unbind(dev.Address); err != nil {
m.logger.Warnf("Failed to unbind device %s: %v", dev.Address, err)
}
}
Expand All @@ -136,9 +138,7 @@ func (m unbindCommand) unbindAll() error {

func (m unbindCommand) unbindDevice() error {
device := m.options.deviceID
// Note: Despite its name, GetGPUByPciBusID returns any NVIDIA PCI device
// (GPU, NVSwitch, etc.) at the specified address, not just GPUs.
nvdev, err := m.nvpci.GetGPUByPciBusID(device)
nvdev, err := m.nvpci.GetNvidiaDeviceByPciBusID(device)
if err != nil {
return fmt.Errorf("failed to get NVIDIA device: %w", err)
}
Expand All @@ -157,7 +157,7 @@ func (m unbindCommand) unbindDevice() error {

m.logger.Infof("Unbinding device %s", device)

if err := m.nvpassthrough.UnbindFromDriver(nvdev); err != nil {
if err := m.nvpassthrough.Unbind(device); err != nil {
return fmt.Errorf("failed to unbind device %s from driver: %w", device, err)
}

Expand Down
7 changes: 5 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ require (
github.com/NVIDIA/go-nvlib v0.10.0
github.com/moby/sys/mount v0.3.4
github.com/sirupsen/logrus v1.9.4
github.com/stretchr/testify v1.11.1
github.com/urfave/cli/v2 v2.27.7
golang.org/x/sys v0.45.0
k8s.io/api v0.36.1
Expand All @@ -15,6 +14,8 @@ require (
k8s.io/kubectl v0.36.1
)

replace github.com/NVIDIA/go-nvlib => ../go-nvlib

require (
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
Expand All @@ -36,6 +37,7 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.18.6 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
Expand All @@ -46,10 +48,11 @@ require (
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/pmorjan/kmod v1.1.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spf13/cobra v1.10.2 // indirect
github.com/spf13/pflag v1.0.9 // indirect
github.com/ulikunitz/xz v0.5.15 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xlab/treeprint v1.2.0 // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
Expand Down
8 changes: 6 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ
github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/NVIDIA/go-nvlib v0.10.0 h1:2jbAFmvLBntIc/4iUChI9DzxyYNI92pohXU4kFuNrg0=
github.com/NVIDIA/go-nvlib v0.10.0/go.mod h1:7mzx9FSdO9fXWP9NKuZmWkCwhkEcSWQFe2tmFwtLb9c=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
Expand Down Expand Up @@ -57,6 +55,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
Expand Down Expand Up @@ -95,6 +95,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmorjan/kmod v1.1.1 h1:Vfw6bMaOg/sYSBCqJPT9TbqHHf5zK00GbaL5JQLO4r0=
github.com/pmorjan/kmod v1.1.1/go.mod h1:jR4fVosEpQ6b5U0rpxaqoShTDPvCjLIP8vEESZyvnqQ=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
Expand All @@ -120,6 +122,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
Expand Down
Loading
Loading