ResiLien/nebula
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move all of tun into overlay (#577)

Browse Source
This commit is contained in:
Nate Brown
2021-11-11 16:37:29 -06:00
committed by GitHub
parent 88ce0edf76
commit e07524a654
18 changed files with 203 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

217
overlay/route.go Normal file
View File

@ -0,0 +1,217 @@
package overlay
import (
"fmt"
"math"
"net"
"strconv"
"github.com/slackhq/nebula/config"
)
type Route struct {
MTU int
Metric int
Cidr *net.IPNet
Via *net.IP
}
func ParseRoutes(c *config.C, network *net.IPNet) ([]Route, error) {
var err error
r := c.Get("tun.routes")
if r == nil {
return []Route{}, nil
}
rawRoutes, ok := r.([]interface{})
if !ok {
return nil, fmt.Errorf("tun.routes is not an array")
}
if len(rawRoutes) < 1 {
return []Route{}, nil
}
routes := make([]Route, len(rawRoutes))
for i, r := range rawRoutes {
m, ok := r.(map[interface{}]interface{})
if !ok {
return nil, fmt.Errorf("entry %v in tun.routes is invalid", i+1)
}
rMtu, ok := m["mtu"]
if !ok {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not present", i+1)
}
mtu, ok := rMtu.(int)
if !ok {
mtu, err = strconv.Atoi(rMtu.(string))
if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, err)
}
}
if mtu < 500 {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is below 500: %v", i+1, mtu)
}
rRoute, ok := m["route"]
if !ok {
return nil, fmt.Errorf("entry %v.route in tun.routes is not present", i+1)
}
r := Route{
MTU: mtu,
}
_, r.Cidr, err = net.ParseCIDR(fmt.Sprintf("%v", rRoute))
if err != nil {
return nil, fmt.Errorf("entry %v.route in tun.routes failed to parse: %v", i+1, err)
}
if !ipWithin(network, r.Cidr) {
return nil, fmt.Errorf(
"entry %v.route in tun.routes is not contained within the network attached to the certificate; route: %v, network: %v",
i+1,
r.Cidr.String(),
network.String(),
)
}
routes[i] = r
}
return routes, nil
}
func ParseUnsafeRoutes(c *config.C, network *net.IPNet) ([]Route, error) {
var err error
r := c.Get("tun.unsafe_routes")
if r == nil {
return []Route{}, nil
}
rawRoutes, ok := r.([]interface{})
if !ok {
return nil, fmt.Errorf("tun.unsafe_routes is not an array")
}
if len(rawRoutes) < 1 {
return []Route{}, nil
}
routes := make([]Route, len(rawRoutes))
for i, r := range rawRoutes {
m, ok := r.(map[interface{}]interface{})
if !ok {
return nil, fmt.Errorf("entry %v in tun.unsafe_routes is invalid", i+1)
}
rMtu, ok := m["mtu"]
if !ok {
rMtu = c.GetInt("tun.mtu", DefaultMTU)
}
mtu, ok := rMtu.(int)
if !ok {
mtu, err = strconv.Atoi(rMtu.(string))
if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err)
}
}
if mtu < 500 {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is below 500: %v", i+1, mtu)
}
rMetric, ok := m["metric"]
if !ok {
rMetric = 0
}
metric, ok := rMetric.(int)
if !ok {
_, err = strconv.ParseInt(rMetric.(string), 10, 32)
if err != nil {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, err)
}
}
if metric < 0 || metric > math.MaxInt32 {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not in range (0-%d) : %v", i+1, math.MaxInt32, metric)
}
rVia, ok := m["via"]
if !ok {
return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes is not present", i+1)
}
via, ok := rVia.(string)
if !ok {
return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes is not a string: found %T", i+1, rVia)
}
nVia := net.ParseIP(via)
if nVia == nil {
return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes failed to parse address: %v", i+1, via)
}
rRoute, ok := m["route"]
if !ok {
return nil, fmt.Errorf("entry %v.route in tun.unsafe_routes is not present", i+1)
}
r := Route{
Via: &nVia,
MTU: mtu,
Metric: metric,
}
_, r.Cidr, err = net.ParseCIDR(fmt.Sprintf("%v", rRoute))
if err != nil {
return nil, fmt.Errorf("entry %v.route in tun.unsafe_routes failed to parse: %v", i+1, err)
}
if ipWithin(network, r.Cidr) {
return nil, fmt.Errorf(
"entry %v.route in tun.unsafe_routes is contained within the network attached to the certificate; route: %v, network: %v",
i+1,
r.Cidr.String(),
network.String(),
)
}
routes[i] = r
}
return routes, nil
}
func ipWithin(o *net.IPNet, i *net.IPNet) bool {
// Make sure o contains the lowest form of i
if !o.Contains(i.IP.Mask(i.Mask)) {
return false
}
// Find the max ip in i
ip4 := i.IP.To4()
if ip4 == nil {
return false
}
last := make(net.IP, len(ip4))
copy(last, ip4)
for x := range ip4 {
last[x] |= ^i.Mask[x]
}
// Make sure o contains the max
if !o.Contains(last) {
return false
}
return true
}

41
overlay/tun.go Normal file
View File

@ -0,0 +1,41 @@
package overlay
import (
"net"
"github.com/sirupsen/logrus"
"github.com/slackhq/nebula/config"
)
const DefaultMTU = 1300
func NewDeviceFromConfig(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, routes, unsafeRoutes []Route, fd *int, routines int) (Device, error) {
switch {
case c.GetBool("tun.disabled", false):
tun := newDisabledTun(tunCidr, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
return tun, nil
case fd != nil:
return newTunFromFd(
l,
*fd,
tunCidr,
c.GetInt("tun.mtu", DefaultMTU),
routes,
unsafeRoutes,
c.GetInt("tun.tx_queue", 500),
)
default:
return newTun(
l,
c.GetString("tun.dev", ""),
tunCidr,
c.GetInt("tun.mtu", DefaultMTU),
routes,
unsafeRoutes,
c.GetInt("tun.tx_queue", 500),
routines > 1,
)
}
}

86
overlay/tun_android.go Normal file
View File

@ -0,0 +1,86 @@
//go:build !e2e_testing
// +build !e2e_testing
package overlay
import (
"fmt"
"io"
"net"
"os"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
)
type Tun struct {
io.ReadWriteCloser
fd int
Device string
Cidr *net.IPNet
MaxMTU int
DefaultMTU int
TXQueueLen int
Routes []Route
UnsafeRoutes []Route
l *logrus.Logger
}
func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int) (ifce *Tun, err error) {
file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
ifce = &Tun{
ReadWriteCloser: file,
fd: int(file.Fd()),
Device: "android",
Cidr: cidr,
DefaultMTU: defaultMTU,
TXQueueLen: txQueueLen,
Routes: routes,
UnsafeRoutes: unsafeRoutes,
l: l,
}
return
}
func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
return nil, fmt.Errorf("newTun not supported in Android")
}
func (c *Tun) WriteRaw(b []byte) error {
var nn int
for {
max := len(b)
n, err := unix.Write(c.fd, b[nn:max])
if n > 0 {
nn += n
}
if nn == len(b) {
return err
}
if err != nil {
return err
}
if n == 0 {
return io.ErrUnexpectedEOF
}
}
}
func (c Tun) Activate() error {
return nil
}
func (c *Tun) CidrNet() *net.IPNet {
return c.Cidr
}
func (c *Tun) DeviceName() string {
return c.Device
}
func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
}

412
overlay/tun_darwin.go Normal file
View File

@ -0,0 +1,412 @@
//go:build !ios && !e2e_testing
// +build !ios,!e2e_testing
package overlay
import (
"fmt"
"io"
"net"
"os"
"syscall"
"unsafe"
"github.com/sirupsen/logrus"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
type Tun struct {
io.ReadWriteCloser
Device string
Cidr *net.IPNet
DefaultMTU int
TXQueueLen int
UnsafeRoutes []Route
l *logrus.Logger
// cache out buffer since we need to prepend 4 bytes for tun metadata
out []byte
}
type sockaddrCtl struct {
scLen uint8
scFamily uint8
ssSysaddr uint16
scID uint32
scUnit uint32
scReserved [5]uint32
}
type ifReq struct {
Name [16]byte
Flags uint16
pad [8]byte
}
func ioctl(a1, a2, a3 uintptr) error {
_, _, errno := unix.Syscall(unix.SYS_IOCTL, a1, a2, a3)
if errno != 0 {
return errno
}
return nil
}
var sockaddrCtlSize uintptr = 32
const (
_SYSPROTO_CONTROL = 2 //define SYSPROTO_CONTROL 2 /* kernel control protocol */
_AF_SYS_CONTROL = 2 //#define AF_SYS_CONTROL 2 /* corresponding sub address type */
_PF_SYSTEM = unix.AF_SYSTEM //#define PF_SYSTEM AF_SYSTEM
_CTLIOCGINFO = 3227799043 //#define CTLIOCGINFO _IOWR('N', 3, struct ctl_info)
utunControlName = "com.apple.net.utun_control"
)
type ifreqAddr struct {
Name [16]byte
Addr unix.RawSockaddrInet4
pad [8]byte
}
type ifreqMTU struct {
Name [16]byte
MTU int32
pad [8]byte
}
type ifreqQLEN struct {
Name [16]byte
Value int32
pad [8]byte
}
func newTun(l *logrus.Logger, name string, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
if len(routes) > 0 {
return nil, fmt.Errorf("route MTU not supported in Darwin")
}
ifIndex := -1
if name != "" && name != "utun" {
_, err := fmt.Sscanf(name, "utun%d", &ifIndex)
if err != nil || ifIndex < 0 {
// NOTE: we don't make this error so we don't break existing
// configs that set a name before it was used.
l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring")
ifIndex = -1
}
}
fd, err := unix.Socket(_PF_SYSTEM, unix.SOCK_DGRAM, _SYSPROTO_CONTROL)
if err != nil {
return nil, fmt.Errorf("system socket: %v", err)
}
var ctlInfo = &struct {
ctlID uint32
ctlName [96]byte
}{}
copy(ctlInfo.ctlName[:], []byte(utunControlName))
err = ioctl(uintptr(fd), uintptr(_CTLIOCGINFO), uintptr(unsafe.Pointer(ctlInfo)))
if err != nil {
return nil, fmt.Errorf("CTLIOCGINFO: %v", err)
}
sc := sockaddrCtl{
scLen: uint8(sockaddrCtlSize),
scFamily: unix.AF_SYSTEM,
ssSysaddr: _AF_SYS_CONTROL,
scID: ctlInfo.ctlID,
scUnit: uint32(ifIndex) + 1,
}
_, _, errno := unix.RawSyscall(
unix.SYS_CONNECT,
uintptr(fd),
uintptr(unsafe.Pointer(&sc)),
uintptr(sockaddrCtlSize),
)
if errno != 0 {
return nil, fmt.Errorf("SYS_CONNECT: %v", errno)
}
var ifName struct {
name [16]byte
}
ifNameSize := uintptr(len(ifName.name))
_, _, errno = syscall.Syscall6(syscall.SYS_GETSOCKOPT, uintptr(fd),
2, // SYSPROTO_CONTROL
2, // UTUN_OPT_IFNAME
uintptr(unsafe.Pointer(&ifName)),
uintptr(unsafe.Pointer(&ifNameSize)), 0)
if errno != 0 {
return nil, fmt.Errorf("SYS_GETSOCKOPT: %v", errno)
}
name = string(ifName.name[:ifNameSize-1])
err = syscall.SetNonblock(fd, true)
if err != nil {
return nil, fmt.Errorf("SetNonblock: %v", err)
}
file := os.NewFile(uintptr(fd), "")
tun := &Tun{
ReadWriteCloser: file,
Device: name,
Cidr: cidr,
DefaultMTU: defaultMTU,
TXQueueLen: txQueueLen,
UnsafeRoutes: unsafeRoutes,
l: l,
}
return tun, nil
}
func (t *Tun) deviceBytes() (o [16]byte) {
for i, c := range t.Device {
o[i] = byte(c)
}
return
}
func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int) (ifce *Tun, err error) {
return nil, fmt.Errorf("newTunFromFd not supported in Darwin")
}
func (c *Tun) Close() error {
if c.ReadWriteCloser != nil {
return c.ReadWriteCloser.Close()
}
return nil
}
func (t *Tun) Activate() error {
devName := t.deviceBytes()
var addr, mask [4]byte
copy(addr[:], t.Cidr.IP.To4())
copy(mask[:], t.Cidr.Mask)
s, err := unix.Socket(
unix.AF_INET,
unix.SOCK_DGRAM,
unix.IPPROTO_IP,
)
if err != nil {
return err
}
fd := uintptr(s)
ifra := ifreqAddr{
Name: devName,
Addr: unix.RawSockaddrInet4{
Family: unix.AF_INET,
Addr: addr,
},
}
// Set the device ip address
if err = ioctl(fd, unix.SIOCSIFADDR, uintptr(unsafe.Pointer(&ifra))); err != nil {
return fmt.Errorf("failed to set tun address: %s", err)
}
// Set the device network
ifra.Addr.Addr = mask
if err = ioctl(fd, unix.SIOCSIFNETMASK, uintptr(unsafe.Pointer(&ifra))); err != nil {
return fmt.Errorf("failed to set tun netmask: %s", err)
}
// Set the device name
ifrf := ifReq{Name: devName}
if err = ioctl(fd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
return fmt.Errorf("failed to set tun device name: %s", err)
}
// Set the MTU on the device
ifm := ifreqMTU{Name: devName, MTU: int32(t.DefaultMTU)}
if err = ioctl(fd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
return fmt.Errorf("Failed to set tun mtu: %v", err)
}
/*
// Set the transmit queue length
ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)}
if err = ioctl(fd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil {
// If we can't set the queue length nebula will still work but it may lead to packet loss
l.WithError(err).Error("Failed to set tun tx queue length")
}
*/
// Bring up the interface
ifrf.Flags = ifrf.Flags | unix.IFF_UP
if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
return fmt.Errorf("failed to bring the tun device up: %s", err)
}
routeSock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
}
defer func() {
unix.Shutdown(routeSock, unix.SHUT_RDWR)
err := unix.Close(routeSock)
if err != nil {
t.l.WithError(err).Error("failed to close AF_ROUTE socket")
}
}()
routeAddr := &netroute.Inet4Addr{}
maskAddr := &netroute.Inet4Addr{}
linkAddr, err := getLinkAddr(t.Device)
if err != nil {
return err
}
if linkAddr == nil {
return fmt.Errorf("unable to discover link_addr for tun interface")
}
copy(routeAddr.IP[:], addr[:])
copy(maskAddr.IP[:], mask[:])
err = addRoute(routeSock, routeAddr, maskAddr, linkAddr)
if err != nil {
return err
}
// Run the interface
ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
return fmt.Errorf("failed to run tun device: %s", err)
}
// Unsafe path routes
for _, r := range t.UnsafeRoutes {
copy(routeAddr.IP[:], r.Cidr.IP.To4())
copy(maskAddr.IP[:], net.IP(r.Cidr.Mask).To4())
err = addRoute(routeSock, routeAddr, maskAddr, linkAddr)
if err != nil {
return err
}
// TODO how to set metric
}
return nil
}
// Get the LinkAddr for the interface of the given name
// TODO: Is there an easier way to fetch this when we create the interface?
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
func getLinkAddr(name string) (*netroute.LinkAddr, error) {
rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0)
if err != nil {
return nil, err
}
msgs, err := netroute.ParseRIB(unix.NET_RT_IFLIST, rib)
if err != nil {
return nil, err
}
for _, m := range msgs {
switch m := m.(type) {
case *netroute.InterfaceMessage:
if m.Name == name {
sa, ok := m.Addrs[unix.RTAX_IFP].(*netroute.LinkAddr)
if ok {
return sa, nil
}
}
}
}
return nil, nil
}
func addRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) error {
r := netroute.RouteMessage{
Version: unix.RTM_VERSION,
Type: unix.RTM_ADD,
Flags: unix.RTF_UP,
Seq: 1,
Addrs: []netroute.Addr{
unix.RTAX_DST: addr,
unix.RTAX_GATEWAY: link,
unix.RTAX_NETMASK: mask,
},
}
data, err := r.Marshal()
if err != nil {
return fmt.Errorf("failed to create route.RouteMessage: %v", err)
}
_, err = unix.Write(sock, data[:])
if err != nil {
return fmt.Errorf("failed to write route.RouteMessage to socket: %v", err)
}
return nil
}
var _ io.ReadWriteCloser = (*Tun)(nil)
func (t *Tun) Read(to []byte) (int, error) {
buf := make([]byte, len(to)+4)
n, err := t.ReadWriteCloser.Read(buf)
copy(to, buf[4:])
return n - 4, err
}
// Write is only valid for single threaded use
func (t *Tun) Write(from []byte) (int, error) {
buf := t.out
if cap(buf) < len(from)+4 {
buf = make([]byte, len(from)+4)
t.out = buf
}
buf = buf[:len(from)+4]
if len(from) == 0 {
return 0, syscall.EIO
}
// Determine the IP Family for the NULL L2 Header
ipVer := from[0] >> 4
if ipVer == 4 {
buf[3] = syscall.AF_INET
} else if ipVer == 6 {
buf[3] = syscall.AF_INET6
} else {
return 0, fmt.Errorf("Unable to determine IP version from packet")
}
copy(buf[4:], from)
n, err := t.ReadWriteCloser.Write(buf)
return n - 4, err
}
func (c *Tun) CidrNet() *net.IPNet {
return c.Cidr
}
func (c *Tun) DeviceName() string {
return c.Device
}
func (c *Tun) WriteRaw(b []byte) error {
_, err := c.Write(b)
return err
}
func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
}

175
overlay/tun_disabled.go Normal file
View File

@ -0,0 +1,175 @@
package overlay
import (
"encoding/binary"
"fmt"
"io"
"net"
"strings"
"github.com/rcrowley/go-metrics"
"github.com/sirupsen/logrus"
)
type disabledTun struct {
read chan []byte
cidr *net.IPNet
// Track these metrics since we don't have the tun device to do it for us
tx metrics.Counter
rx metrics.Counter
l *logrus.Logger
}
func newDisabledTun(cidr *net.IPNet, queueLen int, metricsEnabled bool, l *logrus.Logger) *disabledTun {
tun := &disabledTun{
cidr: cidr,
read: make(chan []byte, queueLen),
l: l,
}
if metricsEnabled {
tun.tx = metrics.GetOrRegisterCounter("messages.tx.message", nil)
tun.rx = metrics.GetOrRegisterCounter("messages.rx.message", nil)
} else {
tun.tx = &metrics.NilCounter{}
tun.rx = &metrics.NilCounter{}
}
return tun
}
func (*disabledTun) Activate() error {
return nil
}
func (t *disabledTun) CidrNet() *net.IPNet {
return t.cidr
}
func (*disabledTun) DeviceName() string {
return "disabled"
}
func (t *disabledTun) Read(b []byte) (int, error) {
r, ok := <-t.read
if !ok {
return 0, io.EOF
}
if len(r) > len(b) {
return 0, fmt.Errorf("packet larger than mtu: %d > %d bytes", len(r), len(b))
}
t.tx.Inc(1)
if t.l.Level >= logrus.DebugLevel {
t.l.WithField("raw", prettyPacket(r)).Debugf("Write payload")
}
return copy(b, r), nil
}
func (t *disabledTun) handleICMPEchoRequest(b []byte) bool {
// Return early if this is not a simple ICMP Echo Request
//TODO: make constants out of these
if !(len(b) >= 28 && len(b) <= 9001 && b[0] == 0x45 && b[9] == 0x01 && b[20] == 0x08) {
return false
}
// We don't support fragmented packets
if b[7] != 0 || (b[6]&0x2F != 0) {
return false
}
buf := make([]byte, len(b))
copy(buf, b)
// Swap dest / src IPs and recalculate checksum
ipv4 := buf[0:20]
copy(ipv4[12:16], b[16:20])
copy(ipv4[16:20], b[12:16])
ipv4[10] = 0
ipv4[11] = 0
binary.BigEndian.PutUint16(ipv4[10:], ipChecksum(ipv4))
// Change type to ICMP Echo Reply and recalculate checksum
icmp := buf[20:]
icmp[0] = 0
icmp[2] = 0
icmp[3] = 0
binary.BigEndian.PutUint16(icmp[2:], ipChecksum(icmp))
// attempt to write it, but don't block
select {
case t.read <- buf:
default:
t.l.Debugf("tun_disabled: dropped ICMP Echo Reply response")
}
return true
}
func (t *disabledTun) Write(b []byte) (int, error) {
t.rx.Inc(1)
// Check for ICMP Echo Request before spending time doing the full parsing
if t.handleICMPEchoRequest(b) {
if t.l.Level >= logrus.DebugLevel {
t.l.WithField("raw", prettyPacket(b)).Debugf("Disabled tun responded to ICMP Echo Request")
}
} else if t.l.Level >= logrus.DebugLevel {
t.l.WithField("raw", prettyPacket(b)).Debugf("Disabled tun received unexpected payload")
}
return len(b), nil
}
func (t *disabledTun) WriteRaw(b []byte) error {
_, err := t.Write(b)
return err
}
func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return t, nil
}
func (t *disabledTun) Close() error {
if t.read != nil {
close(t.read)
t.read = nil
}
return nil
}
type prettyPacket []byte
func (p prettyPacket) String() string {
var s strings.Builder
for i, b := range p {
if i > 0 && i%8 == 0 {
s.WriteString(" ")
}
s.WriteString(fmt.Sprintf("%02x ", b))
}
return s.String()
}
func ipChecksum(b []byte) uint16 {
var c uint32
sz := len(b) - 1
for i := 0; i < sz; i += 2 {
c += uint32(b[i]) << 8
c += uint32(b[i+1])
}
if sz%2 == 0 {
c += uint32(b[sz]) << 8
}
for (c >> 16) > 0 {
c = (c & 0xffff) + (c >> 16)
}
return ^uint16(c)
}

107
overlay/tun_freebsd.go Normal file
View File

@ -0,0 +1,107 @@
//go:build !e2e_testing
// +build !e2e_testing
package overlay
import (
"fmt"
"io"
"net"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
"github.com/sirupsen/logrus"
)
var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
type Tun struct {
Device string
Cidr *net.IPNet
MTU int
UnsafeRoutes []Route
l *logrus.Logger
io.ReadWriteCloser
}
func (c *Tun) Close() error {
if c.ReadWriteCloser != nil {
return c.ReadWriteCloser.Close()
}
return nil
}
func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int) (ifce *Tun, err error) {
return nil, fmt.Errorf("newTunFromFd not supported in FreeBSD")
}
func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
if len(routes) > 0 {
return nil, fmt.Errorf("Route MTU not supported in FreeBSD")
}
if strings.HasPrefix(deviceName, "/dev/") {
deviceName = strings.TrimPrefix(deviceName, "/dev/")
}
if !deviceNameRE.MatchString(deviceName) {
return nil, fmt.Errorf("tun.dev must match `tun[0-9]+`")
}
return &Tun{
Device: deviceName,
Cidr: cidr,
MTU: defaultMTU,
UnsafeRoutes: unsafeRoutes,
l: l,
}, nil
}
func (c *Tun) Activate() error {
var err error
c.ReadWriteCloser, err = os.OpenFile("/dev/"+c.Device, os.O_RDWR, 0)
if err != nil {
return fmt.Errorf("Activate failed: %v", err)
}
// TODO use syscalls instead of exec.Command
c.l.Debug("command: ifconfig", c.Device, c.Cidr.String(), c.Cidr.IP.String())
if err = exec.Command("/sbin/ifconfig", c.Device, c.Cidr.String(), c.Cidr.IP.String()).Run(); err != nil {
return fmt.Errorf("failed to run 'ifconfig': %s", err)
}
c.l.Debug("command: route", "-n", "add", "-net", c.Cidr.String(), "-interface", c.Device)
if err = exec.Command("/sbin/route", "-n", "add", "-net", c.Cidr.String(), "-interface", c.Device).Run(); err != nil {
return fmt.Errorf("failed to run 'route add': %s", err)
}
c.l.Debug("command: ifconfig", c.Device, "mtu", strconv.Itoa(c.MTU))
if err = exec.Command("/sbin/ifconfig", c.Device, "mtu", strconv.Itoa(c.MTU)).Run(); err != nil {
return fmt.Errorf("failed to run 'ifconfig': %s", err)
}
// Unsafe path routes
for _, r := range c.UnsafeRoutes {
c.l.Debug("command: route", "-n", "add", "-net", r.Cidr.String(), "-interface", c.Device)
if err = exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), "-interface", c.Device).Run(); err != nil {
return fmt.Errorf("failed to run 'route add' for unsafe_route %s: %s", r.Cidr.String(), err)
}
}
return nil
}
func (c *Tun) CidrNet() *net.IPNet {
return c.Cidr
}
func (c *Tun) DeviceName() string {
return c.Device
}
func (c *Tun) WriteRaw(b []byte) error {
_, err := c.Write(b)
return err
}
func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
}

120
overlay/tun_ios.go Normal file
View File

@ -0,0 +1,120 @@
//go:build ios && !e2e_testing
// +build ios,!e2e_testing
package overlay
import (
"errors"
"fmt"
"io"
"net"
"os"
"sync"
"syscall"
"github.com/sirupsen/logrus"
)
type Tun struct {
io.ReadWriteCloser
Device string
Cidr *net.IPNet
}
func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
return nil, fmt.Errorf("newTun not supported in iOS")
}
func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int) (ifce *Tun, err error) {
if len(routes) > 0 {
return nil, fmt.Errorf("route MTU not supported in Darwin")
}
file := os.NewFile(uintptr(deviceFd), "/dev/tun")
ifce = &Tun{
Cidr: cidr,
Device: "iOS",
ReadWriteCloser: &tunReadCloser{f: file},
}
return
}
func (c *Tun) Activate() error {
return nil
}
func (c *Tun) WriteRaw(b []byte) error {
_, err := c.Write(b)
return err
}
// The following is hoisted up from water, we do this so we can inject our own fd on iOS
type tunReadCloser struct {
f io.ReadWriteCloser
rMu sync.Mutex
rBuf []byte
wMu sync.Mutex
wBuf []byte
}
func (t *tunReadCloser) Read(to []byte) (int, error) {
t.rMu.Lock()
defer t.rMu.Unlock()
if cap(t.rBuf) < len(to)+4 {
t.rBuf = make([]byte, len(to)+4)
}
t.rBuf = t.rBuf[:len(to)+4]
n, err := t.f.Read(t.rBuf)
copy(to, t.rBuf[4:])
return n - 4, err
}
func (t *tunReadCloser) Write(from []byte) (int, error) {
if len(from) == 0 {
return 0, syscall.EIO
}
t.wMu.Lock()
defer t.wMu.Unlock()
if cap(t.wBuf) < len(from)+4 {
t.wBuf = make([]byte, len(from)+4)
}
t.wBuf = t.wBuf[:len(from)+4]
// Determine the IP Family for the NULL L2 Header
ipVer := from[0] >> 4
if ipVer == 4 {
t.wBuf[3] = syscall.AF_INET
} else if ipVer == 6 {
t.wBuf[3] = syscall.AF_INET6
} else {
return 0, errors.New("unable to determine IP version from packet")
}
copy(t.wBuf[4:], from)
n, err := t.f.Write(t.wBuf)
return n - 4, err
}
func (t *tunReadCloser) Close() error {
return t.f.Close()
}
func (c *Tun) CidrNet() *net.IPNet {
return c.Cidr
}
func (c *Tun) DeviceName() string {
return c.Device
}
func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
}

342
overlay/tun_linux.go Normal file
View File

@ -0,0 +1,342 @@
//go:build !android && !e2e_testing
// +build !android,!e2e_testing
package overlay
import (
"fmt"
"io"
"net"
"os"
"strings"
"unsafe"
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
)
type Tun struct {
io.ReadWriteCloser
fd int
Device string
Cidr *net.IPNet
MaxMTU int
DefaultMTU int
TXQueueLen int
Routes []Route
UnsafeRoutes []Route
l *logrus.Logger
}
type ifReq struct {
Name [16]byte
Flags uint16
pad [8]byte
}
func ioctl(a1, a2, a3 uintptr) error {
_, _, errno := unix.Syscall(unix.SYS_IOCTL, a1, a2, a3)
if errno != 0 {
return errno
}
return nil
}
/*
func ipv4(addr string) (o [4]byte, err error) {
ip := net.ParseIP(addr).To4()
if ip == nil {
err = fmt.Errorf("failed to parse addr %s", addr)
return
}
for i, b := range ip {
o[i] = b
}
return
}
*/
const (
cIFF_TUN = 0x0001
cIFF_NO_PI = 0x1000
cIFF_MULTI_QUEUE = 0x0100
)
type ifreqAddr struct {
Name [16]byte
Addr unix.RawSockaddrInet4
pad [8]byte
}
type ifreqMTU struct {
Name [16]byte
MTU int32
pad [8]byte
}
type ifreqQLEN struct {
Name [16]byte
Value int32
pad [8]byte
}
func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int) (ifce *Tun, err error) {
file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
ifce = &Tun{
ReadWriteCloser: file,
fd: int(file.Fd()),
Device: "tun0",
Cidr: cidr,
DefaultMTU: defaultMTU,
TXQueueLen: txQueueLen,
Routes: routes,
UnsafeRoutes: unsafeRoutes,
l: l,
}
return
}
func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
if err != nil {
return nil, err
}
var req ifReq
req.Flags = uint16(cIFF_TUN | cIFF_NO_PI)
if multiqueue {
req.Flags |= cIFF_MULTI_QUEUE
}
copy(req.Name[:], deviceName)
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
return nil, err
}
name := strings.Trim(string(req.Name[:]), "\x00")
file := os.NewFile(uintptr(fd), "/dev/net/tun")
maxMTU := defaultMTU
for _, r := range routes {
if r.MTU > maxMTU {
maxMTU = r.MTU
}
}
ifce = &Tun{
ReadWriteCloser: file,
fd: int(file.Fd()),
Device: name,
Cidr: cidr,
MaxMTU: maxMTU,
DefaultMTU: defaultMTU,
TXQueueLen: txQueueLen,
Routes: routes,
UnsafeRoutes: unsafeRoutes,
l: l,
}
return
}
func (c *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
if err != nil {
return nil, err
}
var req ifReq
req.Flags = uint16(cIFF_TUN | cIFF_NO_PI | cIFF_MULTI_QUEUE)
copy(req.Name[:], c.Device)
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
return nil, err
}
file := os.NewFile(uintptr(fd), "/dev/net/tun")
return file, nil
}
func (c *Tun) WriteRaw(b []byte) error {
var nn int
for {
max := len(b)
n, err := unix.Write(c.fd, b[nn:max])
if n > 0 {
nn += n
}
if nn == len(b) {
return err
}
if err != nil {
return err
}
if n == 0 {
return io.ErrUnexpectedEOF
}
}
}
func (c *Tun) Write(b []byte) (int, error) {
return len(b), c.WriteRaw(b)
}
func (c Tun) deviceBytes() (o [16]byte) {
for i, c := range c.Device {
o[i] = byte(c)
}
return
}
func (c Tun) Activate() error {
devName := c.deviceBytes()
var addr, mask [4]byte
copy(addr[:], c.Cidr.IP.To4())
copy(mask[:], c.Cidr.Mask)
s, err := unix.Socket(
unix.AF_INET,
unix.SOCK_DGRAM,
unix.IPPROTO_IP,
)
if err != nil {
return err
}
fd := uintptr(s)
ifra := ifreqAddr{
Name: devName,
Addr: unix.RawSockaddrInet4{
Family: unix.AF_INET,
Addr: addr,
},
}
// Set the device ip address
if err = ioctl(fd, unix.SIOCSIFADDR, uintptr(unsafe.Pointer(&ifra))); err != nil {
return fmt.Errorf("failed to set tun address: %s", err)
}
// Set the device network
ifra.Addr.Addr = mask
if err = ioctl(fd, unix.SIOCSIFNETMASK, uintptr(unsafe.Pointer(&ifra))); err != nil {
return fmt.Errorf("failed to set tun netmask: %s", err)
}
// Set the device name
ifrf := ifReq{Name: devName}
if err = ioctl(fd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
return fmt.Errorf("failed to set tun device name: %s", err)
}
// Set the MTU on the device
ifm := ifreqMTU{Name: devName, MTU: int32(c.MaxMTU)}
if err = ioctl(fd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
// This is currently a non fatal condition because the route table must have the MTU set appropriately as well
c.l.WithError(err).Error("Failed to set tun mtu")
}
// Set the transmit queue length
ifrq := ifreqQLEN{Name: devName, Value: int32(c.TXQueueLen)}
if err = ioctl(fd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil {
// If we can't set the queue length nebula will still work but it may lead to packet loss
c.l.WithError(err).Error("Failed to set tun tx queue length")
}
// Bring up the interface
ifrf.Flags = ifrf.Flags | unix.IFF_UP
if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
return fmt.Errorf("failed to bring the tun device up: %s", err)
}
// Set the routes
link, err := netlink.LinkByName(c.Device)
if err != nil {
return fmt.Errorf("failed to get tun device link: %s", err)
}
// Default route
dr := &net.IPNet{IP: c.Cidr.IP.Mask(c.Cidr.Mask), Mask: c.Cidr.Mask}
nr := netlink.Route{
LinkIndex: link.Attrs().Index,
Dst: dr,
MTU: c.DefaultMTU,
AdvMSS: c.advMSS(Route{}),
Scope: unix.RT_SCOPE_LINK,
Src: c.Cidr.IP,
Protocol: unix.RTPROT_KERNEL,
Table: unix.RT_TABLE_MAIN,
Type: unix.RTN_UNICAST,
}
err = netlink.RouteReplace(&nr)
if err != nil {
return fmt.Errorf("failed to set mtu %v on the default route %v; %v", c.DefaultMTU, dr, err)
}
// Path routes
for _, r := range c.Routes {
nr := netlink.Route{
LinkIndex: link.Attrs().Index,
Dst: r.Cidr,
MTU: r.MTU,
AdvMSS: c.advMSS(r),
Scope: unix.RT_SCOPE_LINK,
}
err = netlink.RouteAdd(&nr)
if err != nil {
return fmt.Errorf("failed to set mtu %v on route %v; %v", r.MTU, r.Cidr, err)
}
}
// Unsafe path routes
for _, r := range c.UnsafeRoutes {
nr := netlink.Route{
LinkIndex: link.Attrs().Index,
Dst: r.Cidr,
MTU: r.MTU,
Priority: r.Metric,
AdvMSS: c.advMSS(r),
Scope: unix.RT_SCOPE_LINK,
}
err = netlink.RouteAdd(&nr)
if err != nil {
return fmt.Errorf("failed to set mtu %v on route %v; %v", r.MTU, r.Cidr, err)
}
}
// Run the interface
ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
return fmt.Errorf("failed to run tun device: %s", err)
}
return nil
}
func (c *Tun) CidrNet() *net.IPNet {
return c.Cidr
}
func (c *Tun) DeviceName() string {
return c.Device
}
func (c Tun) advMSS(r Route) int {
mtu := r.MTU
if r.MTU == 0 {
mtu = c.DefaultMTU
}
// We only need to set advmss if the route MTU does not match the device MTU
if mtu != c.MaxMTU {
return mtu - 40
}
return 0
}

34
overlay/tun_linux_test.go Normal file
View File

@ -0,0 +1,34 @@
//go:build !e2e_testing
// +build !e2e_testing
package overlay
import "testing"
var runAdvMSSTests = []struct {
name string
tun Tun
r Route
expected int
}{
// Standard case, default MTU is the device max MTU
{"default", Tun{DefaultMTU: 1440, MaxMTU: 1440}, Route{}, 0},
{"default-min", Tun{DefaultMTU: 1440, MaxMTU: 1440}, Route{MTU: 1440}, 0},
{"default-low", Tun{DefaultMTU: 1440, MaxMTU: 1440}, Route{MTU: 1200}, 1160},
// Case where we have a route MTU set higher than the default
{"route", Tun{DefaultMTU: 1440, MaxMTU: 8941}, Route{}, 1400},
{"route-min", Tun{DefaultMTU: 1440, MaxMTU: 8941}, Route{MTU: 1440}, 1400},
{"route-high", Tun{DefaultMTU: 1440, MaxMTU: 8941}, Route{MTU: 8941}, 0},
}
func TestTunAdvMSS(t *testing.T) {
for _, tt := range runAdvMSSTests {
t.Run(tt.name, func(t *testing.T) {
o := tt.tun.advMSS(tt.r)
if o != tt.expected {
t.Errorf("got %d, want %d", o, tt.expected)
}
})
}
}

237
overlay/tun_test.go Normal file
View File

@ -0,0 +1,237 @@
package overlay
import (
"fmt"
"net"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
)
func Test_ParseRoutes(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
_, n, _ := net.ParseCIDR("10.0.0.0/24")
// test no routes config
routes, err := ParseRoutes(c, n)
assert.Nil(t, err)
assert.Len(t, routes, 0)
// not an array
c.Settings["tun"] = map[interface{}]interface{}{"routes": "hi"}
routes, err = ParseRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "tun.routes is not an array")
// no routes
c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{}}
routes, err = ParseRoutes(c, n)
assert.Nil(t, err)
assert.Len(t, routes, 0)
// weird route
c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{"asdf"}}
routes, err = ParseRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1 in tun.routes is invalid")
// no mtu
c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{}}}
routes, err = ParseRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.mtu in tun.routes is not present")
// bad mtu
c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "nope"}}}
routes, err = ParseRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.mtu in tun.routes is not an integer: strconv.Atoi: parsing \"nope\": invalid syntax")
// low mtu
c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "499"}}}
routes, err = ParseRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.mtu in tun.routes is below 500: 499")
// missing route
c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "500"}}}
routes, err = ParseRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.route in tun.routes is not present")
// unparsable route
c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "route": "nope"}}}
routes, err = ParseRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.route in tun.routes failed to parse: invalid CIDR address: nope")
// below network range
c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "route": "1.0.0.0/8"}}}
routes, err = ParseRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.route in tun.routes is not contained within the network attached to the certificate; route: 1.0.0.0/8, network: 10.0.0.0/24")
// above network range
c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "route": "10.0.1.0/24"}}}
routes, err = ParseRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.route in tun.routes is not contained within the network attached to the certificate; route: 10.0.1.0/24, network: 10.0.0.0/24")
// happy case
c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{
map[interface{}]interface{}{"mtu": "9000", "route": "10.0.0.0/29"},
map[interface{}]interface{}{"mtu": "8000", "route": "10.0.0.1/32"},
}}
routes, err = ParseRoutes(c, n)
assert.Nil(t, err)
assert.Len(t, routes, 2)
tested := 0
for _, r := range routes {
if r.MTU == 8000 {
assert.Equal(t, "10.0.0.1/32", r.Cidr.String())
tested++
} else {
assert.Equal(t, 9000, r.MTU)
assert.Equal(t, "10.0.0.0/29", r.Cidr.String())
tested++
}
}
if tested != 2 {
t.Fatal("Did not see both routes")
}
}
func Test_ParseUnsafeRoutes(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
_, n, _ := net.ParseCIDR("10.0.0.0/24")
// test no routes config
routes, err := ParseUnsafeRoutes(c, n)
assert.Nil(t, err)
assert.Len(t, routes, 0)
// not an array
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": "hi"}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "tun.unsafe_routes is not an array")
// no routes
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, err)
assert.Len(t, routes, 0)
// weird route
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{"asdf"}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1 in tun.unsafe_routes is invalid")
// no via
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{}}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.via in tun.unsafe_routes is not present")
// invalid via
for _, invalidValue := range []interface{}{
127, false, nil, 1.0, []string{"1", "2"},
} {
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": invalidValue}}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, fmt.Sprintf("entry 1.via in tun.unsafe_routes is not a string: found %T", invalidValue))
}
// unparsable via
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "via": "nope"}}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.via in tun.unsafe_routes failed to parse address: nope")
// missing route
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "500"}}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.route in tun.unsafe_routes is not present")
// unparsable route
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "500", "route": "nope"}}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.route in tun.unsafe_routes failed to parse: invalid CIDR address: nope")
// within network range
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "route": "10.0.0.0/24"}}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.route in tun.unsafe_routes is contained within the network attached to the certificate; route: 10.0.0.0/24, network: 10.0.0.0/24")
// below network range
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "route": "1.0.0.0/8"}}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Len(t, routes, 1)
assert.Nil(t, err)
// above network range
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "route": "10.0.1.0/24"}}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Len(t, routes, 1)
assert.Nil(t, err)
// no mtu
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "route": "1.0.0.0/8"}}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Len(t, routes, 1)
assert.Equal(t, DefaultMTU, routes[0].MTU)
// bad mtu
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "nope"}}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is not an integer: strconv.Atoi: parsing \"nope\": invalid syntax")
// low mtu
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "499"}}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, routes)
assert.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is below 500: 499")
// happy case
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{
map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "9000", "route": "1.0.0.0/29"},
map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "8000", "route": "1.0.0.1/32"},
map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "1500", "metric": 1234, "route": "1.0.0.2/32"},
}}
routes, err = ParseUnsafeRoutes(c, n)
assert.Nil(t, err)
assert.Len(t, routes, 3)
tested := 0
for _, r := range routes {
if r.MTU == 8000 {
assert.Equal(t, "1.0.0.1/32", r.Cidr.String())
tested++
} else if r.MTU == 9000 {
assert.Equal(t, 9000, r.MTU)
assert.Equal(t, "1.0.0.0/29", r.Cidr.String())
tested++
} else {
assert.Equal(t, 1500, r.MTU)
assert.Equal(t, 1234, r.Metric)
assert.Equal(t, "1.0.0.2/32", r.Cidr.String())
tested++
}
}
if tested != 3 {
t.Fatal("Did not see both unsafe_routes")
}
}

105
overlay/tun_tester.go Normal file
View File

@ -0,0 +1,105 @@
//go:build e2e_testing
// +build e2e_testing
package overlay
import (
"fmt"
"io"
"net"
"github.com/sirupsen/logrus"
)
type Tun struct {
Device string
Cidr *net.IPNet
MTU int
UnsafeRoutes []Route
l *logrus.Logger
rxPackets chan []byte // Packets to receive into nebula
TxPackets chan []byte // Packets transmitted outside by nebula
}
func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, _ []Route, unsafeRoutes []Route, _ int, _ bool) (ifce *Tun, err error) {
return &Tun{
Device: deviceName,
Cidr: cidr,
MTU: defaultMTU,
UnsafeRoutes: unsafeRoutes,
l: l,
rxPackets: make(chan []byte, 1),
TxPackets: make(chan []byte, 1),
}, nil
}
func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ []Route, _ int) (ifce *Tun, err error) {
return nil, fmt.Errorf("newTunFromFd not supported")
}
// Send will place a byte array onto the receive queue for nebula to consume
// These are unencrypted ip layer frames destined for another nebula node.
// packets should exit the udp side, capture them with udpConn.Get
func (c *Tun) Send(packet []byte) {
c.l.WithField("dataLen", len(packet)).Info("Tun receiving injected packet")
c.rxPackets <- packet
}
// Get will pull an unencrypted ip layer frame from the transmit queue
// nebula meant to send this message to some application on the local system
// packets were ingested from the udp side, you can send them with udpConn.Send
func (c *Tun) Get(block bool) []byte {
if block {
return <-c.TxPackets
}
select {
case p := <-c.TxPackets:
return p
default:
return nil
}
}
//********************************************************************************************************************//
// Below this is boilerplate implementation to make nebula actually work
//********************************************************************************************************************//
func (c *Tun) Activate() error {
return nil
}
func (c *Tun) CidrNet() *net.IPNet {
return c.Cidr
}
func (c *Tun) DeviceName() string {
return c.Device
}
func (c *Tun) Write(b []byte) (n int, err error) {
return len(b), c.WriteRaw(b)
}
func (c *Tun) Close() error {
close(c.rxPackets)
return nil
}
func (c *Tun) WriteRaw(b []byte) error {
packet := make([]byte, len(b), len(b))
copy(packet, b)
c.TxPackets <- packet
return nil
}
func (c *Tun) Read(b []byte) (int, error) {
p := <-c.rxPackets
copy(b, p)
return len(p), nil
}
func (c *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return nil, fmt.Errorf("TODO: multiqueue not implemented")
}

107
overlay/tun_water_windows.go Normal file
View File

@ -0,0 +1,107 @@
package overlay
import (
"fmt"
"io"
"net"
"os/exec"
"strconv"
"github.com/songgao/water"
)
type WindowsWaterTun struct {
Device string
Cidr *net.IPNet
MTU int
UnsafeRoutes []Route
*water.Interface
}
func newWindowsWaterTun(deviceName string, cidr *net.IPNet, defaultMTU int, unsafeRoutes []Route, txQueueLen int) (ifce *WindowsWaterTun, err error) {
// NOTE: You cannot set the deviceName under Windows, so you must check tun.Device after calling .Activate()
return &WindowsWaterTun{
Cidr: cidr,
MTU: defaultMTU,
UnsafeRoutes: unsafeRoutes,
}, nil
}
func (c *WindowsWaterTun) Activate() error {
var err error
c.Interface, err = water.New(water.Config{
DeviceType: water.TUN,
PlatformSpecificParams: water.PlatformSpecificParams{
ComponentID: "tap0901",
Network: c.Cidr.String(),
},
})
if err != nil {
return fmt.Errorf("Activate failed: %v", err)
}
c.Device = c.Interface.Name()
// TODO use syscalls instead of exec.Command
err = exec.Command(
`C:\Windows\System32\netsh.exe`, "interface", "ipv4", "set", "address",
fmt.Sprintf("name=%s", c.Device),
"source=static",
fmt.Sprintf("addr=%s", c.Cidr.IP),
fmt.Sprintf("mask=%s", net.IP(c.Cidr.Mask)),
"gateway=none",
).Run()
if err != nil {
return fmt.Errorf("failed to run 'netsh' to set address: %s", err)
}
err = exec.Command(
`C:\Windows\System32\netsh.exe`, "interface", "ipv4", "set", "interface",
c.Device,
fmt.Sprintf("mtu=%d", c.MTU),
).Run()
if err != nil {
return fmt.Errorf("failed to run 'netsh' to set MTU: %s", err)
}
iface, err := net.InterfaceByName(c.Device)
if err != nil {
return fmt.Errorf("failed to find interface named %s: %v", c.Device, err)
}
for _, r := range c.UnsafeRoutes {
err = exec.Command(
"C:\\Windows\\System32\\route.exe", "add", r.Cidr.String(), r.Via.String(), "IF", strconv.Itoa(iface.Index), "METRIC", strconv.Itoa(r.Metric),
).Run()
if err != nil {
return fmt.Errorf("failed to add the unsafe_route %s: %v", r.Cidr.String(), err)
}
}
return nil
}
func (c *WindowsWaterTun) CidrNet() *net.IPNet {
return c.Cidr
}
func (c *WindowsWaterTun) DeviceName() string {
return c.Device
}
func (c *WindowsWaterTun) WriteRaw(b []byte) error {
_, err := c.Write(b)
return err
}
func (c *WindowsWaterTun) Close() error {
if c.Interface == nil {
return nil
}
return c.Interface.Close()
}
func (t *WindowsWaterTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
}

74
overlay/tun_windows.go Normal file
View File

@ -0,0 +1,74 @@
//go:build !e2e_testing
// +build !e2e_testing
package overlay
import (
"fmt"
"io"
"net"
"os"
"path/filepath"
"runtime"
"syscall"
"github.com/sirupsen/logrus"
)
type Tun struct {
Device
}
func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int) (ifce *Tun, err error) {
return nil, fmt.Errorf("newTunFromFd not supported in Windows")
}
func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, unsafeRoutes []Route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) {
if len(routes) > 0 {
return nil, fmt.Errorf("route MTU not supported in Windows")
}
useWintun := true
if err = checkWinTunExists(); err != nil {
l.WithError(err).Warn("Check Wintun driver failed, fallback to wintap driver")
useWintun = false
}
var inside Device
if useWintun {
inside, err = newWinTun(deviceName, cidr, defaultMTU, unsafeRoutes, txQueueLen)
if err != nil {
return nil, fmt.Errorf("Create Wintun interface failed, %w", err)
}
} else {
inside, err = newWindowsWaterTun(deviceName, cidr, defaultMTU, unsafeRoutes, txQueueLen)
if err != nil {
return nil, fmt.Errorf("Create wintap driver failed, %w", err)
}
}
return &Tun{
Device: inside,
}, nil
}
func checkWinTunExists() error {
myPath, err := os.Executable()
if err != nil {
return err
}
arch := runtime.GOARCH
switch arch {
case "386":
//NOTE: wintun bundles 386 as x86
arch = "x86"
}
_, err = syscall.LoadDLL(filepath.Join(filepath.Dir(myPath), "dist", "windows", "wintun", "bin", arch, "wintun.dll"))
return err
}
func (t *Tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
}

153
overlay/tun_wintun_windows.go Normal file
View File

@ -0,0 +1,153 @@
package overlay
import (
"crypto"
"fmt"
"io"
"net"
"unsafe"
"github.com/slackhq/nebula/wintun"
"golang.org/x/sys/windows"
"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
)
const tunGUIDLabel = "Fixed Nebula Windows GUID v1"
type WinTun struct {
Device string
Cidr *net.IPNet
MTU int
UnsafeRoutes []Route
tun *wintun.NativeTun
}
func generateGUIDByDeviceName(name string) (*windows.GUID, error) {
// GUID is 128 bit
hash := crypto.MD5.New()
_, err := hash.Write([]byte(tunGUIDLabel))
if err != nil {
return nil, err
}
_, err = hash.Write([]byte(name))
if err != nil {
return nil, err
}
sum := hash.Sum(nil)
return (*windows.GUID)(unsafe.Pointer(&sum[0])), nil
}
func newWinTun(deviceName string, cidr *net.IPNet, defaultMTU int, unsafeRoutes []Route, txQueueLen int) (ifce *WinTun, err error) {
guid, err := generateGUIDByDeviceName(deviceName)
if err != nil {
return nil, fmt.Errorf("Generate GUID failed: %w", err)
}
tunDevice, err := wintun.CreateTUNWithRequestedGUID(deviceName, guid, defaultMTU)
if err != nil {
return nil, fmt.Errorf("Create TUN device failed: %w", err)
}
ifce = &WinTun{
Device: deviceName,
Cidr: cidr,
MTU: defaultMTU,
UnsafeRoutes: unsafeRoutes,
tun: tunDevice.(*wintun.NativeTun),
}
return ifce, nil
}
func (c *WinTun) Activate() error {
luid := winipcfg.LUID(c.tun.LUID())
if err := luid.SetIPAddresses([]net.IPNet{*c.Cidr}); err != nil {
return fmt.Errorf("failed to set address: %w", err)
}
foundDefault4 := false
routes := make([]*winipcfg.RouteData, 0, len(c.UnsafeRoutes)+1)
for _, r := range c.UnsafeRoutes {
if !foundDefault4 {
if cidr, bits := r.Cidr.Mask.Size(); cidr == 0 && bits != 0 {
foundDefault4 = true
}
}
// Add our unsafe route
routes = append(routes, &winipcfg.RouteData{
Destination: *r.Cidr,
NextHop: *r.Via,
Metric: uint32(r.Metric),
})
}
if err := luid.AddRoutes(routes); err != nil {
return fmt.Errorf("failed to add routes: %w", err)
}
ipif, err := luid.IPInterface(windows.AF_INET)
if err != nil {
return fmt.Errorf("failed to get ip interface: %w", err)
}
ipif.NLMTU = uint32(c.MTU)
if foundDefault4 {
ipif.UseAutomaticMetric = false
ipif.Metric = 0
}
if err := ipif.Set(); err != nil {
return fmt.Errorf("failed to set ip interface: %w", err)
}
return nil
}
func (c *WinTun) CidrNet() *net.IPNet {
return c.Cidr
}
func (c *WinTun) DeviceName() string {
return c.Device
}
func (c *WinTun) Read(b []byte) (int, error) {
return c.tun.Read(b, 0)
}
func (c *WinTun) Write(b []byte) (int, error) {
return c.tun.Write(b, 0)
}
func (c *WinTun) WriteRaw(b []byte) error {
_, err := c.Write(b)
return err
}
func (c *WinTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
}
func (c *WinTun) Close() error {
// It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes,
// so to be certain, just remove everything before destroying.
luid := winipcfg.LUID(c.tun.LUID())
_ = luid.FlushRoutes(windows.AF_INET)
_ = luid.FlushIPAddresses(windows.AF_INET)
/* We don't support IPV6 yet
_ = luid.FlushRoutes(windows.AF_INET6)
_ = luid.FlushIPAddresses(windows.AF_INET6)
*/
_ = luid.FlushDNS(windows.AF_INET)
return c.tun.Close()
}