diff --git a/hypervisor/events.go b/hypervisor/events.go index e8a2f3a8..2302b257 100644 --- a/hypervisor/events.go +++ b/hypervisor/events.go @@ -52,6 +52,7 @@ type InterfaceCreated struct { Id string //user specified in (ref api.InterfaceDescription: a user identifier of interface, user may use this to specify a nic, normally you can use IPAddr as an Id.) Index int PCIAddr int + TapFd int Bridge string HostDevice string DeviceName string @@ -73,6 +74,7 @@ type NetDevInsertedEvent struct { Index int DeviceName string Address int + TapFd int } func (ne *NetDevInsertedEvent) ResultId() string { diff --git a/hypervisor/network.go b/hypervisor/network.go index c7860666..bf07a936 100644 --- a/hypervisor/network.go +++ b/hypervisor/network.go @@ -173,6 +173,8 @@ func (nc *NetworkContext) addInterface(inf *api.InterfaceDescription, result cha result <- fe return } else if ni, ok := ev.(*NetDevInsertedEvent); ok { + created := nc.idMap[inf.Id] + created.TapFd = ni.TapFd nc.sandbox.Log(DEBUG, "nic insert success: %s", ni.Id) result <- ni return diff --git a/hypervisor/qemu/network.go b/hypervisor/qemu/network.go index 9b98f388..dc58f425 100644 --- a/hypervisor/qemu/network.go +++ b/hypervisor/qemu/network.go @@ -4,31 +4,58 @@ package qemu import ( "fmt" + "os" "os/exec" "strings" "syscall" + "unsafe" "github.com/golang/glog" "github.com/hyperhq/runv/hypervisor/network" - "github.com/vishvananda/netlink" ) -func GetTapDevice(device, bridge, options string) error { - la := netlink.NewLinkAttrs() - la.Name = device - tapdev := &netlink.Tuntap{LinkAttrs: la, Mode: syscall.IFF_TAP} +const ( + IFNAMSIZ = 16 + CIFF_TAP = 0x0002 + CIFF_NO_PI = 0x1000 + CIFF_ONE_QUEUE = 0x2000 +) + +type ifReq struct { + Name [IFNAMSIZ]byte + Flags uint16 + pad [0x28 - 0x10 - 2]byte +} - if err := netlink.LinkAdd(tapdev); err != nil { - glog.Errorf("fail to create tap device: %v, %v", device, err) - return err +func GetTapFd(device, bridge, options string) (int, error) { + var ( + req ifReq + errno syscall.Errno + ) + + tapFile, err := os.OpenFile("/dev/net/tun", os.O_RDWR, 0) + if err != nil { + return -1, err } - if err := network.UpAndAddToBridge(device, bridge, options); err != nil { + req.Flags = CIFF_TAP | CIFF_NO_PI | CIFF_ONE_QUEUE + copy(req.Name[:len(req.Name)-1], []byte(device)) + _, _, errno = syscall.Syscall(syscall.SYS_IOCTL, tapFile.Fd(), + uintptr(syscall.TUNSETIFF), + uintptr(unsafe.Pointer(&req))) + if errno != 0 { + tapFile.Close() + return -1, fmt.Errorf("create tap device failed\n") + } + + err = network.UpAndAddToBridge(device, bridge, options) + if err != nil { glog.Errorf("Add to bridge failed %s %s", bridge, device) - return err + tapFile.Close() + return -1, err } - return nil + return int(tapFile.Fd()), nil } func GetVhostUserPort(device, bridge, sockPath, option string) error { diff --git a/hypervisor/qemu/qemu.go b/hypervisor/qemu/qemu.go index 9ff5e6f3..9b71c84a 100644 --- a/hypervisor/qemu/qemu.go +++ b/hypervisor/qemu/qemu.go @@ -260,13 +260,16 @@ func (qc *QemuContext) RemoveDisk(ctx *hypervisor.VmContext, blockInfo *hypervis } func (qc *QemuContext) AddNic(ctx *hypervisor.VmContext, host *hypervisor.HostNicInfo, guest *hypervisor.GuestNicInfo, result chan<- hypervisor.VmEvent) { - waitChan := make(chan hypervisor.VmEvent, 1) - var err error = nil + var ( + fd int = -1 + err error + waitChan chan hypervisor.VmEvent = make(chan hypervisor.VmEvent, 1) + ) if ctx.Boot.EnableVhostUser { err = GetVhostUserPort(host.Device, host.Bridge, ctx.HomeDir, host.Options) } else { - err = GetTapDevice(host.Device, host.Bridge, host.Options) + fd, err = GetTapFd(host.Device, host.Bridge, host.Options) } if err != nil { @@ -281,15 +284,20 @@ func (qc *QemuContext) AddNic(ctx *hypervisor.VmContext, host *hypervisor.HostNi // close tap file if necessary ev, ok := <-waitChan if !ok { + syscall.Close(fd) close(result) } else { + if _, ok := ev.(*hypervisor.DeviceFailed); ok { + syscall.Close(fd) + } result <- ev } }() - newNetworkAddSession(ctx, qc, host, guest, waitChan) + newNetworkAddSession(ctx, qc, fd, host, guest, waitChan) } func (qc *QemuContext) RemoveNic(ctx *hypervisor.VmContext, n *hypervisor.InterfaceCreated, callback hypervisor.VmEvent, result chan<- hypervisor.VmEvent) { + syscall.Close(n.TapFd) newNetworkDelSession(ctx, qc, n.NewName, callback, result) } diff --git a/hypervisor/qemu/qmp_wrapper_amd64.go b/hypervisor/qemu/qmp_wrapper_amd64.go index c5b3b986..bf840566 100644 --- a/hypervisor/qemu/qmp_wrapper_amd64.go +++ b/hypervisor/qemu/qmp_wrapper_amd64.go @@ -4,11 +4,13 @@ package qemu import ( "fmt" + "syscall" + "github.com/golang/glog" "github.com/hyperhq/runv/hypervisor" ) -func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, host *hypervisor.HostNicInfo, guest *hypervisor.GuestNicInfo, result chan<- hypervisor.VmEvent) { +func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, fd int, host *hypervisor.HostNicInfo, guest *hypervisor.GuestNicInfo, result chan<- hypervisor.VmEvent) { busAddr := fmt.Sprintf("0x%x", guest.Busaddr) commands := []*QmpCommand{} if ctx.Boot.EnableVhostUser { @@ -40,15 +42,26 @@ func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, host *hype "vhostforce": true, }, }) - } else { + } else if fd > 0 { + scm := syscall.UnixRights(fd) + glog.V(1).Infof("send net to qemu at %d", fd) + commands = append(commands, &QmpCommand{ + Execute: "getfd", + Arguments: map[string]interface{}{ + "fdname": "fd" + guest.Device, + }, + Scm: scm, + }, &QmpCommand{ + Execute: "netdev_add", + Arguments: map[string]interface{}{ + "type": "tap", "id": guest.Device, "fd": "fd" + guest.Device, + }, + }) + } else if host.Device != "" { commands = append(commands, &QmpCommand{ Execute: "netdev_add", Arguments: map[string]interface{}{ - "type": "tap", - "script": "no", - "id": guest.Device, - "ifname": host.Device, - "br": host.Bridge, + "type": "tap", "id": guest.Device, "ifname": host.Device, "script": "no", }, }) } @@ -71,6 +84,7 @@ func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, host *hype Index: guest.Index, DeviceName: guest.Device, Address: guest.Busaddr, + TapFd: fd, }), } } diff --git a/hypervisor/qemu/qmp_wrapper_arm64.go b/hypervisor/qemu/qmp_wrapper_arm64.go index 79b83d8b..a713da66 100644 --- a/hypervisor/qemu/qmp_wrapper_arm64.go +++ b/hypervisor/qemu/qmp_wrapper_arm64.go @@ -4,24 +4,31 @@ package qemu import ( "fmt" + "syscall" + "github.com/golang/glog" "github.com/hyperhq/runv/hypervisor" ) -func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, host *hypervisor.HostNicInfo, guest *hypervisor.GuestNicInfo, result chan<- hypervisor.VmEvent) { +func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, fd int, host *hypervisor.HostNicInfo, guest *hypervisor.GuestNicInfo, result chan<- hypervisor.VmEvent) { busAddr := fmt.Sprintf("0x%x", guest.Busaddr) - commands := []*QmpCommand{} - commands = append(commands, &QmpCommand{ + commands := make([]*QmpCommand, 3) + scm := syscall.UnixRights(fd) + glog.V(1).Infof("send net to qemu at %d", fd) + commands[0] = &QmpCommand{ + Execute: "getfd", + Arguments: map[string]interface{}{ + "fdname": "fd" + guest.Device, + }, + Scm: scm, + } + commands[1] = &QmpCommand{ Execute: "netdev_add", Arguments: map[string]interface{}{ - "type": "tap", - "script": "no", - "id": guest.Device, - "ifname": host.Device, - "br": host.Bridge, + "type": "tap", "id": guest.Device, "fd": "fd" + guest.Device, }, - }) - commands = append(commands, &QmpCommand{ + } + commands[2] = &QmpCommand{ Execute: "device_add", Arguments: map[string]interface{}{ "netdev": guest.Device, @@ -33,7 +40,7 @@ func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, host *hype "mac": host.Mac, "id": guest.Device, }, - }) + } qc.qmp <- &QmpSession{ commands: commands, @@ -42,6 +49,7 @@ func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, host *hype Index: guest.Index, DeviceName: guest.Device, Address: guest.Busaddr, + TapFd: fd, }), } } diff --git a/hypervisor/qemu/qmp_wrapper_ppc64le.go b/hypervisor/qemu/qmp_wrapper_ppc64le.go index e02e03fd..749e8081 100644 --- a/hypervisor/qemu/qmp_wrapper_ppc64le.go +++ b/hypervisor/qemu/qmp_wrapper_ppc64le.go @@ -10,20 +10,25 @@ import ( "github.com/hyperhq/runv/hypervisor" ) -func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, qc *QemuContext, host *hypervisor.HostNicInfo, guest *hypervisor.GuestNicInfo, result chan<- hypervisor.VmEvent) { +func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, fd int, host *hypervisor.HostNicInfo, guest *hypervisor.GuestNicInfo, result chan<- hypervisor.VmEvent) { busAddr := fmt.Sprintf("0x%x", guest.Busaddr) - commands := []*QmpCommand{} - commands = append(commands, &QmpCommand{ + commands := make([]*QmpCommand, 3) + scm := syscall.UnixRights(fd) + glog.V(1).Infof("send net to qemu at %d", fd) + commands[0] = &QmpCommand{ + Execute: "getfd", + Arguments: map[string]interface{}{ + "fdname": "fd" + guest.Device, + }, + Scm: scm, + } + commands[1] = &QmpCommand{ Execute: "netdev_add", Arguments: map[string]interface{}{ - "type": "tap", - "script": "no", - "id": guest.Device, - "ifname": host.Device, - "br": host.Bridge, + "type": "tap", "id": guest.Device, "fd": "fd" + guest.Device, }, - }) - commands = append(commands, &QmpCommand{ + } + commands[2] = &QmpCommand{ Execute: "device_add", Arguments: map[string]interface{}{ "driver": "virtio-net-pci", @@ -33,7 +38,7 @@ func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, qc *QemuCo "addr": busAddr, "id": guest.Device, }, - }) + } qc.qmp <- &QmpSession{ commands: commands, @@ -42,6 +47,7 @@ func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, qc *QemuCo Index: guest.Index, DeviceName: guest.Device, Address: guest.Busaddr, + TapFd: fd, }), } } diff --git a/hypervisor/qemu/qmp_wrapper_s390x.go b/hypervisor/qemu/qmp_wrapper_s390x.go index c36890f8..63c0c513 100644 --- a/hypervisor/qemu/qmp_wrapper_s390x.go +++ b/hypervisor/qemu/qmp_wrapper_s390x.go @@ -9,20 +9,24 @@ import ( "github.com/hyperhq/runv/hypervisor" ) -func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, id string, fd int, device, mac string, index, addr int, result chan<- hypervisor.VmEvent) { - commands := []*QmpCommand{} +func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, fd int, host *hypervisor.HostNicInfo, guest *hypervisor.GuestNicInfo, result chan<- hypervisor.VmEvent) { + commands := make([]*QmpCommand, 3) scm := syscall.UnixRights(fd) - commands = appends(commands, &QmpCommand{ + glog.V(1).Infof("send net to qemu at %d", fd) + commands[0] = &QmpCommand{ + Execute: "getfd", + Arguments: map[string]interface{}{ + "fdname": "fd" + guest.Device, + }, + Scm: scm, + } + commands[1] = &QmpCommand{ Execute: "netdev_add", Arguments: map[string]interface{}{ - "type": "tap", - "script": "no", - "id": guest.Device, - "ifname": host.Device, - "br": host.Bridge, + "type": "tap", "id": guest.Device, "fd": "fd" + guest.Device, }, - }) - commands = append(commands, &QmpCommand{ + } + commands[2] = &QmpCommand{ Execute: "device_add", Arguments: map[string]interface{}{ "driver": "virtio-net-ccw", @@ -30,7 +34,7 @@ func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, id string, "mac": host.Mac, "id": guest.Device, }, - }) + } qc.qmp <- &QmpSession{ commands: commands, @@ -39,6 +43,7 @@ func newNetworkAddSession(ctx *hypervisor.VmContext, qc *QemuContext, id string, Index: guest.Index, DeviceName: guest.Device, Address: guest.Busaddr, + TapFd: fd, }), } }