-
Notifications
You must be signed in to change notification settings - Fork 148
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add pci address lock for allocated devices (#220)
* Create allocation interface and implementation. This is needed to lock the allocation of the same PCI address until the cmdDel is called or the kernel remove the network namespace. Signed-off-by: Sebastian Sch <[email protected]> * Use the allocator interface to prevent allocation of still in used pci addresses Fixes: #219 Signed-off-by: Sebastian Sch <[email protected]> Signed-off-by: Sebastian Sch <[email protected]>
- Loading branch information
Showing
7 changed files
with
302 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| package utils | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/containernetworking/plugins/pkg/ns" | ||
| "os" | ||
| "path/filepath" | ||
| ) | ||
|
|
||
| type PCIAllocation interface { | ||
| SaveAllocatedPCI(string, string) error | ||
| DeleteAllocatedPCI(string) error | ||
| IsAllocated(string) error | ||
| } | ||
|
|
||
| type PCIAllocator struct { | ||
| dataDir string | ||
| } | ||
|
|
||
| // NewPCIAllocator returns a new PCI allocator | ||
| // it will use the <dataDir>/pci folder to store the information about allocated PCI addresses | ||
| func NewPCIAllocator(dataDir string) *PCIAllocator { | ||
| return &PCIAllocator{dataDir: filepath.Join(dataDir, "pci")} | ||
| } | ||
|
|
||
| // SaveAllocatedPCI creates a file with the pci address as a name and the network namespace as the content | ||
| // return error if the file was not created | ||
| func (p *PCIAllocator) SaveAllocatedPCI(pciAddress, ns string) error { | ||
| if err := os.MkdirAll(p.dataDir, 0600); err != nil { | ||
| return fmt.Errorf("failed to create the sriov data directory(%q): %v", p.dataDir, err) | ||
| } | ||
|
|
||
| path := filepath.Join(p.dataDir, pciAddress) | ||
| err := os.WriteFile(path, []byte(ns), 0600) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to write used PCI address lock file in the path(%q): %v", path, err) | ||
| } | ||
|
|
||
| return err | ||
| } | ||
|
|
||
| // DeleteAllocatedPCI Remove the allocated PCI file | ||
| // return error if the file doesn't exist | ||
| func (p *PCIAllocator) DeleteAllocatedPCI(pciAddress string) error { | ||
| path := filepath.Join(p.dataDir, pciAddress) | ||
| if err := os.Remove(path); err != nil { | ||
| return fmt.Errorf("error removing PCI address lock file %s: %v", path, err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // IsAllocated checks if the PCI address file exist | ||
| // if it exists we also check the network namespace still exist if not we delete the allocation | ||
| // The function will return an error if the pci is still allocated to a running pod | ||
| func (p *PCIAllocator) IsAllocated(pciAddress string) (bool, error) { | ||
| path := filepath.Join(p.dataDir, pciAddress) | ||
| _, err := os.Stat(path) | ||
| if err != nil { | ||
| if os.IsNotExist(err) { | ||
| return false, nil | ||
| } | ||
|
|
||
| return false, fmt.Errorf("failed to check for pci address file for %s: %v", path, err) | ||
| } | ||
|
|
||
| dat, err := os.ReadFile(path) | ||
| if err != nil { | ||
| return false, fmt.Errorf("failed to read for pci address file for %s: %v", path, err) | ||
| } | ||
|
|
||
| // To prevent a locking of a PCI address for every pciAddress file we also add the netns path where it's been used | ||
| // This way if for some reason the cmdDel command was not called but the pod namespace doesn't exist anymore | ||
| // we release the PCI address | ||
| networkNamespace, err := ns.GetNS(string(dat)) | ||
| if err != nil { | ||
| err = p.DeleteAllocatedPCI(pciAddress) | ||
| if err != nil { | ||
| return false, fmt.Errorf("error deleting the pci allocation for vf pci address %s: %v", pciAddress, err) | ||
| } | ||
|
|
||
| return false, nil | ||
| } | ||
|
|
||
| // Close the network namespace | ||
| networkNamespace.Close() | ||
| return true, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| package utils | ||
|
|
||
| import ( | ||
| . "github.com/onsi/ginkgo" | ||
| . "github.com/onsi/gomega" | ||
|
|
||
| "github.com/containernetworking/plugins/pkg/ns" | ||
| "github.com/containernetworking/plugins/pkg/testutils" | ||
| ) | ||
|
|
||
| var _ = Describe("PCIAllocator", func() { | ||
| var targetNetNS ns.NetNS | ||
| var err error | ||
|
|
||
| AfterEach(func() { | ||
| if targetNetNS != nil { | ||
| targetNetNS.Close() | ||
| err = testutils.UnmountNS(targetNetNS) | ||
| } | ||
| }) | ||
|
|
||
| Context("IsAllocated", func() { | ||
| It("Assuming is not allocated", func() { | ||
| allocator := NewPCIAllocator(ts.dirRoot) | ||
| isAllocated, err := allocator.IsAllocated("0000:af:00.1") | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| Expect(isAllocated).To(BeFalse()) | ||
| }) | ||
|
|
||
| It("Assuming is allocated and namespace exist", func() { | ||
| targetNetNS, err = testutils.NewNS() | ||
| Expect(err).NotTo(HaveOccurred()) | ||
| allocator := NewPCIAllocator(ts.dirRoot) | ||
|
|
||
| err = allocator.SaveAllocatedPCI("0000:af:00.1", targetNetNS.Path()) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| isAllocated, err := allocator.IsAllocated("0000:af:00.1") | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| Expect(isAllocated).To(BeTrue()) | ||
| }) | ||
|
|
||
| It("Assuming is allocated and namespace doesn't exist", func() { | ||
| targetNetNS, err = testutils.NewNS() | ||
| Expect(err).NotTo(HaveOccurred()) | ||
|
|
||
| allocator := NewPCIAllocator(ts.dirRoot) | ||
| err = allocator.SaveAllocatedPCI("0000:af:00.1", targetNetNS.Path()) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| err = targetNetNS.Close() | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| err = testutils.UnmountNS(targetNetNS) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| isAllocated, err := allocator.IsAllocated("0000:af:00.1") | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| Expect(isAllocated).To(BeFalse()) | ||
| }) | ||
| }) | ||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters