Skip to content

Commit

Permalink
Sonic redis with serialized interface reconfiguration (#81)
Browse files Browse the repository at this point in the history
  • Loading branch information
majst01 authored May 16, 2023
1 parent 5275e9c commit bf31824
Show file tree
Hide file tree
Showing 44 changed files with 2,125 additions and 160 deletions.
4 changes: 4 additions & 0 deletions cmd/internal/core/reconfigure-switch.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ func (c *Core) buildSwitcherConfig(s *models.V1SwitchResponse) (*types.Conf, err
}
continue
}

// Firewall-Port
if nic.Vrf == "default" {
fw := &types.Firewall{
Expand Down Expand Up @@ -149,6 +150,8 @@ func (c *Core) buildSwitcherConfig(s *models.V1SwitchResponse) (*types.Conf, err
p.Vrfs[nic.Vrf] = vrf
}
switcherConfig.Ports = p

c.nos.SanitizeConfig(switcherConfig)
switcherConfig.FillRouteMapsAndIPPrefixLists()
m, err := vlan.ReadMapping()
if err != nil {
Expand All @@ -158,6 +161,7 @@ func (c *Core) buildSwitcherConfig(s *models.V1SwitchResponse) (*types.Conf, err
if err != nil {
return nil, err
}

return switcherConfig, nil
}

Expand Down
6 changes: 4 additions & 2 deletions cmd/internal/core/reconfigure-switch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/stretchr/testify/require"

"github.com/metal-stack/metal-core/cmd/internal/switcher/cumulus"
"github.com/metal-stack/metal-core/cmd/internal/switcher/types"
"github.com/metal-stack/metal-go/api/models"
)
Expand All @@ -18,6 +19,7 @@ func TestBuildSwitcherConfig(t *testing.T) {
loopbackIP: "10.0.0.1",
spineUplinks: []string{"swp31", "swp32"},
additionalBridgeVIDs: []string{"201-256", "301-356"},
nos: &cumulus.Cumulus{},
}

n1 := "swp1"
Expand Down Expand Up @@ -71,7 +73,7 @@ func TestBuildSwitcherConfig(t *testing.T) {
IPPrefixLists: []types.IPPrefixList{
{
Name: "vrf104001-in-prefixes",
Spec: "permit 10.244.0.0/16 le 32",
Spec: "permit 10.240.0.0/12 le 32",
},
},
RouteMaps: []types.RouteMap{
Expand All @@ -83,7 +85,7 @@ func TestBuildSwitcherConfig(t *testing.T) {
},
},
},
Cidrs: []string{"10.244.0.0/16"},
Cidrs: []string{"10.240.0.0/12"},
}},
},
AdditionalBridgeVIDs: []string{"201-256", "301-356"},
Expand Down
56 changes: 22 additions & 34 deletions cmd/internal/core/register-switch.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,11 @@ package core

import (
"fmt"
"net"
"os"
"time"

"go.uber.org/zap"
"github.com/avast/retry-go/v4"

"github.com/metal-stack/metal-core/cmd/internal/switcher"
sw "github.com/metal-stack/metal-go/api/client/switch_operations"
"github.com/metal-stack/metal-go/api/models"
)
Expand All @@ -24,7 +22,26 @@ func (c *Core) RegisterSwitch() error {
managementUser string
)

if nics, err = getNics(c.log, c.nos, c.additionalBridgePorts); err != nil {
err = retry.Do(
func() error {
initialized, err := c.nos.IsInitialized()
if err != nil {
return err
}
if initialized {
return nil
}
return fmt.Errorf("switch is not yet initialized")
},
retry.Attempts(120),
retry.Delay(1*time.Second),
retry.DelayType(retry.FixedDelay),
)
if err != nil {
return fmt.Errorf("unable to register switch because it is not initialized: %w", err)
}

if nics, err = c.nos.GetNics(c.log, c.additionalBridgePorts); err != nil {
return fmt.Errorf("unable to get nics: %w", err)
}

Expand Down Expand Up @@ -52,6 +69,7 @@ func (c *Core) RegisterSwitch() error {
ManagementUser: managementUser,
}

// TODO could be done with retry-go
for {
_, _, err := c.driver.SwitchOperations().RegisterSwitch(params, nil)
if err == nil {
Expand All @@ -63,33 +81,3 @@ func (c *Core) RegisterSwitch() error {
c.log.Infow("register switch completed")
return nil
}

func getNics(log *zap.SugaredLogger, nos switcher.NOS, blacklist []string) ([]*models.V1SwitchNic, error) {
var nics []*models.V1SwitchNic
ifs, err := nos.GetSwitchPorts()
if err != nil {
return nil, fmt.Errorf("unable to get all ifs: %w", err)
}
links:
for _, iface := range ifs {
name := iface.Name
mac := iface.HardwareAddr.String()
for _, b := range blacklist {
if b == name {
log.Debugw("skip interface, because it is contained in the blacklist", "interface", name, "blacklist", blacklist)
continue links
}
}
_, err := net.ParseMAC(mac)
if err != nil {
log.Debugw("skip interface with invalid mac", "interface", name, "MAC", mac)
continue
}
nic := &models.V1SwitchNic{
Mac: &mac,
Name: &name,
}
nics = append(nics, nic)
}
return nics, nil
}
2 changes: 1 addition & 1 deletion cmd/internal/switcher/cumulus/applier.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func NewFrrApplier(tplPath string) *templates.Applier {
Dest: frr,
Reloader: reloadFrr,
Tmp: frrTmp,
Tpl: templates.FrrTemplate(tplPath),
Tpl: templates.CumulusFrrTemplate(tplPath),
ValidationService: frrValidationService,
}
}
Expand Down
44 changes: 40 additions & 4 deletions cmd/internal/switcher/cumulus/cumulus.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"strings"

"go.uber.org/zap"
"golang.org/x/exp/slices"

"github.com/metal-stack/metal-core/cmd/internal"
"github.com/metal-stack/metal-core/cmd/internal/switcher/templates"
Expand Down Expand Up @@ -37,6 +38,40 @@ func (c *Cumulus) Apply(cfg *types.Conf) error {
return c.frrApplier.Apply(cfg)
}

func (c *Cumulus) IsInitialized() (initialized bool, err error) {
// FIXME decide how we can detect initialization is complete.
return true, nil
}

func (c *Cumulus) GetNics(log *zap.SugaredLogger, blacklist []string) (nics []*models.V1SwitchNic, err error) {
ifs, err := c.GetSwitchPorts()
if err != nil {
return nil, fmt.Errorf("unable to get all ifs: %w", err)
}

for _, iface := range ifs {
name := iface.Name
mac := iface.HardwareAddr.String()
if slices.Contains(blacklist, name) {
log.Debugw("skip interface, because it is contained in the blacklist", "interface", name, "blacklist", blacklist)
continue
}

if _, err := net.ParseMAC(mac); err != nil {
log.Debugw("skip interface with invalid mac", "interface", name, "MAC", mac)
continue
}

nic := &models.V1SwitchNic{
Mac: &mac,
Name: &name,
}
nics = append(nics, nic)
}

return nics, nil
}

func (c *Cumulus) GetSwitchPorts() ([]*net.Interface, error) {
ifs, err := net.Interfaces()
if err != nil {
Expand All @@ -47,17 +82,18 @@ func (c *Cumulus) GetSwitchPorts() ([]*net.Interface, error) {
for i := range ifs {
iface := &ifs[i]
if !strings.HasPrefix(iface.Name, "swp") {
c.log.Debug("skip interface, because only swp* interface are front panels",
zap.String("interface", iface.Name),
zap.String("MAC", iface.HardwareAddr.String()),
)
c.log.Debugw("skip interface, because only swp* interface are front panels", "interface", iface.Name)
continue
}
switchPorts = append(switchPorts, iface)
}
return switchPorts, nil
}

func (c *Cumulus) SanitizeConfig(cfg *types.Conf) {
// nothing required here
}

func (c *Cumulus) GetOS() (*models.V1SwitchOS, error) {
version := "unknown"
lsbReleaseBytes, err := os.ReadFile("/etc/lsb-release")
Expand Down
19 changes: 17 additions & 2 deletions cmd/internal/switcher/nos.go
Original file line number Diff line number Diff line change
@@ -1,22 +1,37 @@
package switcher

import (
"fmt"
"net"
"os"

"go.uber.org/zap"

"github.com/metal-stack/metal-core/cmd/internal/switcher/cumulus"
"github.com/metal-stack/metal-core/cmd/internal/switcher/sonic"
"github.com/metal-stack/metal-core/cmd/internal/switcher/types"
"github.com/metal-stack/metal-go/api/models"
)

type NOS interface {
SanitizeConfig(cfg *types.Conf)
Apply(cfg *types.Conf) error
IsInitialized() (initialized bool, err error)
GetNics(log *zap.SugaredLogger, blacklist []string) ([]*models.V1SwitchNic, error)
GetSwitchPorts() ([]*net.Interface, error)
GetOS() (*models.V1SwitchOS, error)
GetManagement() (ip, user string, err error)
}

func NewNOS(log *zap.SugaredLogger, frrTplFile, interfacesTplFile string) NOS {
return cumulus.New(log.Named("cumulus"), frrTplFile, interfacesTplFile)
func NewNOS(log *zap.SugaredLogger, frrTplFile, interfacesTplFile string) (NOS, error) {
if _, err := os.Stat(sonic.SonicVersionFile); err == nil {
log.Infow("create sonic NOS")
nos, err := sonic.New(log.Named("sonic"), frrTplFile)
if err != nil {
return nil, fmt.Errorf("failed to initialize SONiC NOS %w", err)
}
return nos, nil
}
log.Infow("create cumulus NOS")
return cumulus.New(log.Named("cumulus"), frrTplFile, interfacesTplFile), nil
}
21 changes: 21 additions & 0 deletions cmd/internal/switcher/sonic/db/appldb.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package db

import (
"context"
)

type ApplDB struct {
c *Client
}

func newApplDB(addr string, id int, sep string) *ApplDB {
return &ApplDB{
c: NewClient(addr, id, sep),
}
}

func (d *ApplDB) ExistPortInitDone(ctx context.Context) (bool, error) {
key := Key{"PORT_TABLE", "PortInitDone"}

return d.c.Exists(ctx, key)
}
67 changes: 67 additions & 0 deletions cmd/internal/switcher/sonic/db/asicdb.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package db

import (
"context"
"errors"

"github.com/redis/go-redis/v9"
)

type AsicDB struct {
c *Client
}

type OID string

func newAsicDB(addr string, id int, sep string) *AsicDB {
return &AsicDB{
c: NewClient(addr, id, sep),
}
}

func (d *AsicDB) GetPortIdBridgePortMap(ctx context.Context) (map[OID]OID, error) {
t := d.c.GetTable(Key{"ASIC_STATE", "SAI_OBJECT_TYPE_BRIDGE_PORT"})

bridges, err := t.GetView(ctx)
if err != nil {
return nil, err
}

m := make(map[OID]OID, len(bridges))
for bridge := range bridges {
port, err := t.HGet(ctx, bridge, "SAI_BRIDGE_PORT_ATTR_PORT_ID")
if err != nil {
if errors.Is(err, redis.Nil) {
continue
}
return nil, err
}
if len(port) == 0 {
continue
}
m[OID(port)] = OID(bridge)
}
return m, nil
}

func (d *AsicDB) ExistBridgePort(ctx context.Context, bridgePort OID) (bool, error) {
key := Key{"ASIC_STATE", "SAI_OBJECT_TYPE_BRIDGE_PORT", string(bridgePort)}

return d.c.Exists(ctx, key)
}

func (d *AsicDB) ExistRouterInterface(ctx context.Context, rif OID) (bool, error) {
key := Key{"ASIC_STATE", "SAI_OBJECT_TYPE_ROUTER_INTERFACE", string(rif)}

return d.c.Exists(ctx, key)
}

func (d *AsicDB) InFecModeRs(ctx context.Context, port OID) (bool, error) {
key := Key{"ASIC_STATE", "SAI_OBJECT_TYPE_PORT", string(port)}

result, err := d.c.HGet(ctx, key, "SAI_PORT_ATTR_FEC_MODE")
if err != nil {
return false, err
}
return result == "SAI_PORT_FEC_MODE_RS", err
}
26 changes: 26 additions & 0 deletions cmd/internal/switcher/sonic/db/asicdb_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package db

import (
"context"
"reflect"
"testing"
)

func TestAsicDB_GetPortIdBridgePortMap(t *testing.T) {
c, mock := NewClientMock(":")
asic := &AsicDB{c: c}
bridgePort := "ASIC_STATE:SAI_OBJECT_TYPE_BRIDGE_PORT:oid:0x3a000000001a0a"
want := map[OID]OID{OID("oid:0x1000000001251"): "oid:0x3a000000001a0a"}

mock.ExpectKeys("ASIC_STATE:SAI_OBJECT_TYPE_BRIDGE_PORT:*").SetVal([]string{bridgePort})
mock.ExpectHGet(bridgePort, "SAI_BRIDGE_PORT_ATTR_PORT_ID").SetVal("oid:0x1000000001251")

got, err := asic.GetPortIdBridgePortMap(context.TODO())
if err != nil {
t.Errorf("GetPortIdBridgePortMap() error = %v, wantErr %v", err, nil)
return
}
if !reflect.DeepEqual(got, want) {
t.Errorf("GetPortIdBridgePortMap() got = %v, want %v", got, want)
}
}
Loading

0 comments on commit bf31824

Please sign in to comment.