diff --git a/Makefile b/Makefile index 4f53b37f3..c2c0a5d31 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,7 @@ TARGETS := \ testdata/errors \ testdata/variables \ testdata/arena \ + testdata/struct_ops \ btf/testdata/relocs \ btf/testdata/relocs_read \ btf/testdata/relocs_read_tgt \ diff --git a/elf_reader.go b/elf_reader.go index f2c9196b7..3f4c802e0 100644 --- a/elf_reader.go +++ b/elf_reader.go @@ -116,8 +116,17 @@ func LoadCollectionSpecFromReader(rd io.ReaderAt) (*CollectionSpec, error) { case sec.Type == elf.SHT_REL: // Store relocations under the section index of the target relSections[elf.SectionIndex(sec.Info)] = sec - case sec.Type == elf.SHT_PROGBITS && (sec.Flags&elf.SHF_EXECINSTR) != 0 && sec.Size > 0: - sections[idx] = newElfSection(sec, programSection) + case sec.Type == elf.SHT_PROGBITS && sec.Size > 0: + if (sec.Flags&elf.SHF_EXECINSTR) != 0 && sec.Size > 0 { + sections[idx] = newElfSection(sec, programSection) + } else if sec.Name == structOpsLinkSec { + // classification based on sec names so that struct_ops-specific + // sections (.struct_ops.link) is correctly recognized + // as non-executable PROGBITS, allowing value placement and link metadata to be loaded. + sections[idx] = newElfSection(sec, structOpsSection) + } else if sec.Name == structOpsSec { + return nil, fmt.Errorf("section %q: got '.struct_ops' section: %w", sec.Name, ErrNotSupported) + } } } @@ -186,6 +195,15 @@ func LoadCollectionSpecFromReader(rd io.ReaderAt) (*CollectionSpec, error) { return nil, fmt.Errorf("load programs: %w", err) } + // assiociate members in structs with ProgramSpecs using relo + if err := ec.associateStructOpsRelocs( + progs, + relSections, + symbols, + ); err != nil { + return nil, fmt.Errorf("load struct_ops: %w", err) + } + return &CollectionSpec{ ec.maps, progs, @@ -239,6 +257,7 @@ const ( btfMapSection programSection dataSection + structOpsSection ) type elfSection struct { @@ -349,6 +368,10 @@ func (ec *elfCode) loadProgramSections() (map[string]*ProgramSpec, error) { continue } + if !(sec.Type == elf.SHT_PROGBITS && (sec.Flags&elf.SHF_EXECINSTR) != 0) { + continue + } + if len(sec.symbols) == 0 { return nil, fmt.Errorf("section %v: missing symbols", sec.Name) } @@ -1379,6 +1402,135 @@ func (ec *elfCode) loadKsymsSection() error { return nil } +// associateStructOpsRelocs handles `.struct_ops.link` +// and associates the target function with the correct struct member in the map. +func (ec *elfCode) associateStructOpsRelocs( + progs map[string]*ProgramSpec, + relSecs map[elf.SectionIndex]*elf.Section, + symbols []elf.Symbol, +) error { + willAttachToMap := make(map[string]bool) + + for secIdx, sec := range ec.sections { + if sec.kind != structOpsSection { + continue + } + + userData, err := sec.Data() + if err != nil { + return fmt.Errorf("failed to read section data: %w", err) + } + + // Resolve the BTF datasec describing variables in this section. + var ds *btf.Datasec + if err := ec.btf.TypeByName(sec.Name, &ds); err != nil { + return fmt.Errorf("datasec %s: %w", sec.Name, err) + } + + // Set flags for .struct_ops.link (BPF_F_LINK). + flags := uint32(0) + if sec.Name == structOpsLinkSec { + flags = sys.BPF_F_LINK + } + + type structOpsMapOfsSize struct { + userOff uint64 + userSize uint64 + } + + ofsSizes := make(map[string]structOpsMapOfsSize) + + for _, vsi := range ds.Vars { + varType := btf.UnderlyingType(vsi.Type).(*btf.Var) + mapName := varType.Name + + userType, ok := btf.UnderlyingType(varType.Type).(*btf.Struct) + if !ok { + return fmt.Errorf("var %s: expect struct, got %T", varType.Name, varType.Type) + } + + userSize := uint64(userType.Size) + userOff := uint64(vsi.Offset) + if userOff+userSize > uint64(len(userData)) { + return fmt.Errorf("%s exceeds section", mapName) + } + + // Register the MapSpec for this struct_ops instance. + ec.maps[mapName] = &MapSpec{ + Name: mapName, + Type: StructOpsMap, + Key: &btf.Int{Size: 4}, + KeySize: 4, + Value: userType, + Flags: flags, + MaxEntries: 1, + Contents: []MapKV{ + { + Key: uint32(0), + Value: append([]byte(nil), userData[userOff:userOff+userSize]...), + }, + }, + } + ofsSizes[mapName] = + structOpsMapOfsSize{userOff, userSize} + } + + // Process relo sections that target this struct_ops section. + for relSecIdx, relSec := range relSecs { + if elf.SectionIndex(relSec.Info) != elf.SectionIndex(secIdx) { + continue + } + + if !(relSec.Type == elf.SHT_REL) { + continue + } + + // Load relocation entries (offset -> symbol). + rels, err := ec.loadSectionRelocations(relSec, symbols) + if err != nil { + return fmt.Errorf("failed to load relocations for section %s (relIdx=%d -> target=%d): %w", + relSec.Name, relSecIdx, secIdx, err) + } + + for relOff, sym := range rels { + var ms *MapSpec + var msName string + var baseOff uint64 + + for mapName, ofsSz := range ofsSizes { + if relOff >= ofsSz.userOff && relOff < ofsSz.userOff+ofsSz.userSize { + baseOff = ofsSz.userOff + ms = ec.maps[mapName] + msName = mapName + break + } + } + + if ms == nil || ms.Type != StructOpsMap { + return fmt.Errorf("struct_ops map %s not found or wrong type", msName) + } + + userSt, ok := btf.As[*btf.Struct](ms.Value) + if !ok { + return fmt.Errorf("map %s value is not a btf.Struct", ms.Name) + } + + moff := btf.Bits((relOff - baseOff) * 8) + if memberName, ok := structOpsFuncPtrMemberAtOffset(userSt, moff); ok { + p, ok := progs[sym.Name] + if !(ok && p.Type == StructOps) { + return fmt.Errorf("program %s not found or not StructOps", sym.Name) + } + p.AttachTo = userSt.Name + ":" + memberName + willAttachToMap[sym.Name] = true + } + } + } + } + + return nil +} + type libbpfElfSectionDef struct { pattern string programType sys.ProgType diff --git a/elf_reader_test.go b/elf_reader_test.go index 9b4fcf4aa..ddc484c86 100644 --- a/elf_reader_test.go +++ b/elf_reader_test.go @@ -12,6 +12,7 @@ import ( "syscall" "testing" + "github.com/cilium/ebpf/asm" "github.com/cilium/ebpf/btf" "github.com/cilium/ebpf/internal" "github.com/cilium/ebpf/internal/kallsyms" @@ -932,6 +933,101 @@ func TestArena(t *testing.T) { mustNewCollection(t, coll, nil) } +func TestStructOps(t *testing.T) { + file := testutils.NativeFile(t, "testdata/struct_ops-%s.elf") + coll, err := LoadCollectionSpec(file) + qt.Assert(t, qt.IsNil(err)) + + userData := []byte{ + // test_1 func ptr (8B) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // test_2 func ptr (8B) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // data (4B) + padding (4B) + 0xef, 0xbe, 0xad, 0xde, 0x00, 0x00, 0x00, 0x00, + } + + want := &CollectionSpec{ + Maps: map[string]*MapSpec{ + "testmod_ops": { + Name: "testmod_ops", + Type: StructOpsMap, + MaxEntries: 1, + Flags: sys.BPF_F_LINK, + Key: &btf.Int{Size: 4}, + KeySize: 4, + Value: &btf.Struct{ + Name: "bpf_testmod_ops", + Size: 24, + Members: []btf.Member{ + { + Name: "test_1", + Type: &btf.Pointer{ + Target: &btf.FuncProto{ + Params: []btf.FuncParam{}, + Return: &btf.Int{Name: "int", Size: 4, Encoding: btf.Signed}}}, + Offset: 0, + }, + { + Name: "test_2", + Type: &btf.Pointer{ + Target: &btf.FuncProto{ + Params: []btf.FuncParam{ + {Type: &btf.Int{Name: "int", Size: 4, Encoding: btf.Signed}}, + {Type: &btf.Int{Name: "int", Size: 4, Encoding: btf.Signed}}, + }, + Return: (*btf.Void)(nil), + }, + }, + Offset: 64, + }, + { + Name: "data", + Type: &btf.Int{Name: "int", Size: 4, Encoding: btf.Signed}, + Offset: 128, // bits + }, + }, + }, + Contents: []MapKV{ + { + Key: uint32(0), + Value: userData, + }, + }, + }, + }, + Programs: map[string]*ProgramSpec{ + "test_1": { + Name: "test_1", + Type: StructOps, + AttachTo: "bpf_testmod_ops:test_1", + License: "GPL", + SectionName: "struct_ops/test_1", + Instructions: asm.Instructions{ + asm.Mov.Imm(asm.R0, 0), + asm.Return(), + }, + }, + }, + Variables: map[string]*VariableSpec{}, + } + + testModOps, ok := coll.Maps["testmod_ops"] + if !ok { + t.Fatalf("testmod_ops doesn't exist") + } + + data, ok := testModOps.Contents[0].Value.([]byte) + if !ok { + t.Fatalf("Contents[0].Value should be an array of byte") + } + + qt.Assert(t, qt.CmpEquals(coll.Programs, want.Programs, csCmpOpts)) + qt.Assert(t, qt.CmpEquals(coll.Maps, want.Maps, csCmpOpts)) + qt.Assert(t, qt.CmpEquals(testModOps.Value, want.Maps["testmod_ops"].Value, csCmpOpts)) + qt.Assert(t, qt.CmpEquals(data, userData, csCmpOpts)) +} + var ( elfPath = flag.String("elfs", os.Getenv("CI_KERNEL_SELFTESTS"), "`Path` containing libbpf-compatible ELFs (defaults to $CI_KERNEL_SELFTESTS)") elfPattern = flag.String("elf-pattern", "*.o", "Glob `pattern` for object files that should be tested") @@ -968,6 +1064,9 @@ func TestLibBPFCompat(t *testing.T) { t.Fatal("Expected an error during load") } } else if err != nil { + if errors.Is(err, errUnknownStructOps) { + t.Skip("Skipping since the struct_ops target doesn't exist in kernel") + } t.Fatal("Error during loading:", err) } } @@ -1061,12 +1160,6 @@ func TestLibBPFCompat(t *testing.T) { } } - for _, ps := range spec.Programs { - if ps.Type == StructOps { - ps.AttachTo = "" - } - } - coreFiles := sourceOfBTF(t, path) if len(coreFiles) == 0 { // NB: test_core_reloc_kernel.o doesn't have dedicated BTF and diff --git a/elf_sections.go b/elf_sections.go index 43dcfb103..660209aa4 100644 --- a/elf_sections.go +++ b/elf_sections.go @@ -104,8 +104,8 @@ var elfSectionDefs = []libbpfElfSectionDef{ {"cgroup/getsockopt", sys.BPF_PROG_TYPE_CGROUP_SOCKOPT, sys.BPF_CGROUP_GETSOCKOPT, _SEC_ATTACHABLE}, {"cgroup/setsockopt", sys.BPF_PROG_TYPE_CGROUP_SOCKOPT, sys.BPF_CGROUP_SETSOCKOPT, _SEC_ATTACHABLE}, {"cgroup/dev", sys.BPF_PROG_TYPE_CGROUP_DEVICE, sys.BPF_CGROUP_DEVICE, _SEC_ATTACHABLE_OPT}, - {"struct_ops+", sys.BPF_PROG_TYPE_STRUCT_OPS, 0, _SEC_NONE}, - {"struct_ops.s+", sys.BPF_PROG_TYPE_STRUCT_OPS, 0, _SEC_SLEEPABLE}, + {"struct_ops+", sys.BPF_PROG_TYPE_STRUCT_OPS, 0, _SEC_NONE | ignoreExtra}, + {"struct_ops.s+", sys.BPF_PROG_TYPE_STRUCT_OPS, 0, _SEC_SLEEPABLE | ignoreExtra}, {"sk_lookup", sys.BPF_PROG_TYPE_SK_LOOKUP, sys.BPF_SK_LOOKUP, _SEC_ATTACHABLE}, {"netfilter", sys.BPF_PROG_TYPE_NETFILTER, sys.BPF_NETFILTER, _SEC_NONE}, } diff --git a/prog.go b/prog.go index 3e724234d..a6ba888c6 100644 --- a/prog.go +++ b/prog.go @@ -37,6 +37,9 @@ var errBadRelocation = errors.New("bad CO-RE relocation") // This error is detected based on heuristics and therefore may not be reliable. var errUnknownKfunc = errors.New("unknown kfunc") +// errUnknownStructOps is returned when the struct_ops target doesn't exist in kernel +var errUnknownStructOps = errors.New("unknown struct_ops target") + // ProgramID represents the unique ID of an eBPF program. type ProgramID = sys.ProgramID diff --git a/struct_ops.go b/struct_ops.go index e70fb779b..17ce6fc52 100644 --- a/struct_ops.go +++ b/struct_ops.go @@ -1,6 +1,7 @@ package ebpf import ( + "errors" "fmt" "reflect" "strings" @@ -10,6 +11,8 @@ import ( ) const structOpsValuePrefix = "bpf_struct_ops_" +const structOpsLinkSec = ".struct_ops.link" +const structOpsSec = ".struct_ops" // structOpsFindInnerType returns the "inner" struct inside a value struct_ops type. // @@ -44,6 +47,9 @@ func structOpsFindTarget(userType *btf.Struct, cache *btf.Cache) (vType *btf.Str target := btf.Type((*btf.Struct)(nil)) spec, module, err := findTargetInKernel(vTypeName, &target, cache) + if errors.Is(err, btf.ErrNotFound) { + return nil, 0, nil, fmt.Errorf("%q doesn't exist in kernel: %w", vTypeName, errUnknownStructOps) + } if err != nil { return nil, 0, nil, fmt.Errorf("lookup value type %q: %w", vTypeName, err) } @@ -118,3 +124,20 @@ func structOpsCopyMember(m, km btf.Member, data []byte, kernVData []byte) error copy(kernVData[dstOff:dstOff+mSize], data[srcOff:srcOff+mSize]) return nil } + +// funcPtrMemberAtOffset returns the member name at bit offset `moff` +// if the member is a pointer to a FuncProto. Otherwise returns an empty string. +func structOpsFuncPtrMemberAtOffset(userSt *btf.Struct, moff btf.Bits) (string, bool) { + for _, m := range userSt.Members { + if m.Offset != moff { + continue + } + mt := btf.UnderlyingType(m.Type) + if ptr, ok := btf.As[*btf.Pointer](mt); ok { + if _, ok := btf.As[*btf.FuncProto](ptr.Target); ok { + return m.Name, true + } + } + } + return "", false +} diff --git a/testdata/struct_ops-eb.elf b/testdata/struct_ops-eb.elf new file mode 100644 index 000000000..d2ea07f79 Binary files /dev/null and b/testdata/struct_ops-eb.elf differ diff --git a/testdata/struct_ops-el.elf b/testdata/struct_ops-el.elf new file mode 100644 index 000000000..0902fd89c Binary files /dev/null and b/testdata/struct_ops-el.elf differ diff --git a/testdata/struct_ops.c b/testdata/struct_ops.c new file mode 100644 index 000000000..d147fcfeb --- /dev/null +++ b/testdata/struct_ops.c @@ -0,0 +1,18 @@ +#include "common.h" + +char _license[] __section("license") = "GPL"; + +struct bpf_testmod_ops { + int (*test_1)(void); + void (*test_2)(int, int); + int data; +}; + +__section("struct_ops/test_1") int test_1(void) { + return 0; +} + +__section(".struct_ops.link") struct bpf_testmod_ops testmod_ops = { + .test_1 = (void *)test_1, + .data = 0xdeadbeef, +};