Skip to content

Commit

Permalink
feat: Add CGroup v2 support
Browse files Browse the repository at this point in the history
Signed-off-by: Steffen Vogel <[email protected]>
  • Loading branch information
stv0g committed Oct 30, 2024
1 parent 373ec7b commit e3dc41a
Show file tree
Hide file tree
Showing 18 changed files with 1,193 additions and 46 deletions.
25 changes: 23 additions & 2 deletions cmd/gontc/gontc.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,19 @@
package main

import (
"context"
"errors"
"flag"
"fmt"
"os"
"path/filepath"
"strings"
"time"

"cunicu.li/gont/v2/internal"
"cunicu.li/gont/v2/internal/utils"
g "cunicu.li/gont/v2/pkg"
"github.com/coreos/go-systemd/v22/dbus"
"golang.org/x/exp/slices"
)

Expand Down Expand Up @@ -174,13 +177,21 @@ func list(args []string) {
}

func clean(args []string) error {
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 10*time.Second)

Check failure on line 181 in cmd/gontc/gontc.go

View workflow job for this annotation

GitHub Actions / lint

lostcancel: the cancel function returned by context.WithTimeout should be called, not discarded, to avoid a context leak (govet)

c, err := dbus.NewWithContext(ctx)
if err != nil {
return fmt.Errorf("failed to connect to D-Bus: %w", err)
}

if len(args) > 1 {
network := args[1]
if err := g.TeardownNetwork(network); err != nil {
if err := g.TeardownNetwork(ctx, c, network); err != nil {
return fmt.Errorf("failed to teardown network '%s': %w", network, err)
}
} else {
return g.TeardownAllNetworks()
return g.TeardownAllNetworks(ctx, c)
}

return nil
Expand All @@ -202,6 +213,16 @@ func exec(network, node string, args []string) error {
return err
}

cgroupName := fmt.Sprintf("gont-run-%d", os.Getpid())
cgroup, err := g.NewCGroup(nil, "scope", cgroupName)
if err != nil {
return fmt.Errorf("failed to create CGroup scope: %w", err)
}

if err := cgroup.Start(); err != nil {
return fmt.Errorf("failed to start CGroup scope: %w", err)
}

return g.Exec(network, node, args)
}

Expand Down
2 changes: 1 addition & 1 deletion default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
buildGoModule {
name = "gont";
src = ./.;
vendorHash = "sha256-IXTpMzTrWRH10vB6hRsMf7ilT5tUG/EPJbYLO+8d9Ik=";
vendorHash = "sha256-EAwP8nNyS6lnLi/OBxxdZzePIiy30l6uFr1Z8SPAllA=";
buildInputs = [ libpcap ];
doCheck = false;
}
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@ module cunicu.li/gont/v2
go 1.23.0

require (
github.com/coreos/go-systemd/v22 v22.5.0
github.com/davecgh/go-spew v1.1.1
github.com/fxamacker/cbor/v2 v2.7.0
github.com/go-delve/delve v1.21.0
github.com/go-ping/ping v1.1.0
github.com/godbus/dbus/v5 v5.1.0
github.com/google/nftables v0.2.0
github.com/gopacket/gopacket v1.3.0
github.com/vishvananda/netlink v1.3.0
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
Expand Down Expand Up @@ -67,6 +69,9 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw=
github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
Expand Down
27 changes: 22 additions & 5 deletions pkg/base_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ type BaseNodeOption interface {

type BaseNode struct {
*Namespace
*CGroup

isHostNode bool
network *Network
Expand All @@ -46,9 +47,7 @@ type BaseNode struct {
logger *zap.Logger
}

func (n *Network) AddNode(name string, opts ...Option) (*BaseNode, error) {
var err error

func (n *Network) AddNode(name string, opts ...Option) (node *BaseNode, err error) {
basePath := filepath.Join(n.VarPath, "nodes", name)
for _, path := range []string{"ns", "files"} {
path = filepath.Join(basePath, path)
Expand All @@ -57,13 +56,22 @@ func (n *Network) AddNode(name string, opts ...Option) (*BaseNode, error) {
}
}

node := &BaseNode{
node = &BaseNode{
name: name,
network: n,
BasePath: basePath,
logger: zap.L().Named("node").With(zap.String("node", name)),
}

cgroupName := fmt.Sprintf("gont-%s-%s", n.Name, name)
if node.CGroup, err = NewCGroup(n.sdConn, "slice", cgroupName, opts...); err != nil {
return nil, fmt.Errorf("failed to create CGroup slice: %w", err)
}

if err := node.CGroup.Start(); err != nil {
return nil, fmt.Errorf("failed to start CGroup slice: %w", err)
}

node.logger.Info("Adding new node")

for _, opt := range opts {
Expand Down Expand Up @@ -121,6 +129,7 @@ func (n *Network) AddNode(name string, opts ...Option) (*BaseNode, error) {
}
}

// Create Netlink connection handle if it does not exist yet
if node.nlHandle == nil {
node.nlHandle, err = nl.NewHandleAt(node.NsHandle)
if err != nil {
Expand Down Expand Up @@ -309,7 +318,15 @@ func (n *BaseNode) Teardown() error {
return err
}

return os.RemoveAll(n.BasePath)
if err := os.RemoveAll(n.BasePath); err != nil {
return err
}

if err := n.CGroup.Stop(); err != nil {
return fmt.Errorf("failed to stop Cgroup slice: %w", err)
}

return nil
}

// WriteProcFS write a value to a path within the ProcFS by entering the namespace of this node.
Expand Down
95 changes: 95 additions & 0 deletions pkg/cgroup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
// SPDX-FileCopyrightText: 2024 Steffen Vogel <[email protected]>
// SPDX-License-Identifier: Apache-2.0

package gont

import (
"context"
"fmt"

"github.com/coreos/go-systemd/v22/dbus"
)

type CGroupOption interface {
ApplyCGroup(s *CGroup)
}

type CGroup struct {
Name string
Type string
Properties []dbus.Property

sdConn *dbus.Conn
}

func NewCGroup(c *dbus.Conn, typ, name string, opts ...Option) (g *CGroup, err error) {
g = &CGroup{
Name: name,
Type: typ,
sdConn: c,
}

// Create D-Bus connection if not existing
if g.sdConn == nil {
if g.sdConn, err = dbus.NewWithContext(context.Background()); err != nil {
return nil, fmt.Errorf("failed to connect to D-Bus: %w", err)
}
}

for _, opt := range opts {
if opt, ok := opt.(CGroupOption); ok {
opt.ApplyCGroup(g)
}
}

return g, nil
}

func (g *CGroup) Unit() string {
return g.Name + "." + g.Type
}

// Start creates the CGroup
func (g *CGroup) Start() error {
if _, err := g.sdConn.StartTransientUnitContext(context.Background(), g.Unit(), "replace", g.Properties, nil); err != nil {
return fmt.Errorf("failed to create slice: %w", err)
}

return nil
}

// Stop stops the CGroup and kills all contained processes
func (g *CGroup) Stop() error {
ch := make(chan string)
if _, err := g.sdConn.StopUnitContext(context.Background(), g.Unit(), "fail", ch); err != nil {
return fmt.Errorf("failed to remove slice: %w", err)
}

if state := <-ch; state != "done" {
return fmt.Errorf("failed to wait for CGroup shutdown: state is %s", state)
}

return nil
}

// Freeze suspends execution of all processes in the control group.
func (g *CGroup) Freeze() error {
return g.sdConn.FreezeUnit(context.Background(), g.Unit())
}

// Thaw resumes execution of all processes in the control group.
func (g *CGroup) Thaw() error {
return g.sdConn.ThawUnit(context.Background(), g.Unit())
}

// SetProperties sets transient systemd CGroup properties of the unit.
// See: https://systemd.io/TRANSIENT-SETTINGS/
func (g *CGroup) SetProperties(opts ...CGroupOption) error {
so := &CGroup{}
for _, opt := range opts {
opt.ApplyCGroup(g)
opt.ApplyCGroup(so)
}

return g.sdConn.SetUnitPropertiesContext(context.Background(), g.Unit(), true, so.Properties...)
}
108 changes: 108 additions & 0 deletions pkg/cgroup_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
// SPDX-FileCopyrightText: 2023 Steffen Vogel <[email protected]>
// SPDX-License-Identifier: Apache-2.0

package gont_test

import (
"fmt"
"testing"
"time"

g "cunicu.li/gont/v2/pkg"
sdo "cunicu.li/gont/v2/pkg/options/systemd"
"github.com/stretchr/testify/require"
)

func TestCGroup(t *testing.T) {
n, err := g.NewNetwork("", globalNetworkOptions...)
require.NoError(t, err, "Failed to create network")
defer n.Close()

h, err := n.AddHost("h1")
require.NoError(t, err)

cmd := h.Command("cat", "/proc/self/cgroup")
out, err := cmd.CombinedOutput()
require.NoError(t, err)

expectedCgroup := fmt.Sprintf("0::/gont.slice/gont-%s.slice/gont-%s-%s.slice/gont-run-%d.scope\n", n.Name, n.Name, h.Name(), cmd.ProcessState.Pid())
require.Equal(t, expectedCgroup, string(out))
}

func TestCGroupPropertyNetwork(t *testing.T) {
n, err := g.NewNetwork("", g.Customize(globalNetworkOptions, sdo.MemoryMax(5<<20))...)
require.NoError(t, err, "Failed to create network")
defer n.Close()

h, err := n.AddHost("h1")
require.NoError(t, err)

cmd := h.Command("bash", "-c", "systemctl show "+n.Unit()+" | grep ^MemoryMax=")
out, err := cmd.CombinedOutput()
require.NoError(t, err)

outExpected := fmt.Sprintf("MemoryMax=%d\n", 5<<20)
require.Equal(t, outExpected, string(out))
}

func TestCGroupPropertyHost(t *testing.T) {
n, err := g.NewNetwork("", globalNetworkOptions...)
require.NoError(t, err, "Failed to create network")
defer n.Close()

h, err := n.AddHost("h1", sdo.MemoryMax(5<<20))
require.NoError(t, err)

cmd := h.Command("bash", "-c", "systemctl show "+h.Unit()+" | grep ^MemoryMax=")
out, err := cmd.CombinedOutput()
require.NoError(t, err)

outExpected := fmt.Sprintf("MemoryMax=%d\n", 5<<20)
require.Equal(t, outExpected, string(out))
}

func TestCGroupPropertyCommand(t *testing.T) {
n, err := g.NewNetwork("", globalNetworkOptions...)
require.NoError(t, err, "Failed to create network")
defer n.Close()

h, err := n.AddHost("h1")
require.NoError(t, err)

cmd := h.Command("bash", "-c", "systemctl show gont-run-$$.scope | grep ^MemoryMax=", sdo.MemoryMax(5<<20))
out, err := cmd.CombinedOutput()
require.NoError(t, err)

outExpected := fmt.Sprintf("MemoryMax=%d\n", 5<<20)
require.Equal(t, outExpected, string(out))
}

func TestCGroupTeardown(t *testing.T) {
n, err := g.NewNetwork("", globalNetworkOptions...)
require.NoError(t, err, "Failed to create network")

h, err := n.AddHost("h1", sdo.MemoryMax(5<<20))
require.NoError(t, err)

cmd := h.Command("sleep", 3600)
err = cmd.Start()
require.NoError(t, err)

exited := make(chan bool)
go func() {
cmd.Wait()

Check failure on line 93 in pkg/cgroup_test.go

View workflow job for this annotation

GitHub Actions / lint

Error return value of `cmd.Wait` is not checked (errcheck)
close(exited)
}()

time.Sleep(10 * time.Millisecond)

err = n.Close()
require.NoError(t, err)

select {
case <-exited:

case <-time.After(10 * time.Millisecond):
require.Fail(t, "Process did not terminate")
}
}
Loading

0 comments on commit e3dc41a

Please sign in to comment.