Refactor remotes and handshaking to give every address a fair shot (#437)

This commit is contained in:
Nathan Brown
2021-04-14 13:50:09 -05:00
committed by GitHub
parent 20bef975cd
commit 710df6a876
25 changed files with 1561 additions and 1385 deletions

View File

@ -9,6 +9,7 @@ import (
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/e2e/router"
"github.com/stretchr/testify/assert"
)
func TestGoodHandshake(t *testing.T) {
@ -23,35 +24,35 @@ func TestGoodHandshake(t *testing.T) {
myControl.Start()
theirControl.Start()
// Send a udp packet through to begin standing up the tunnel, this should come out the other side
t.Log("Send a udp packet through to begin standing up the tunnel, this should come out the other side")
myControl.InjectTunUDPPacket(theirVpnIp, 80, 80, []byte("Hi from me"))
// Have them consume my stage 0 packet. They have a tunnel now
t.Log("Have them consume my stage 0 packet. They have a tunnel now")
theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
// Get their stage 1 packet so that we can play with it
t.Log("Get their stage 1 packet so that we can play with it")
stage1Packet := theirControl.GetFromUDP(true)
// I consume a garbage packet with a proper nebula header for our tunnel
t.Log("I consume a garbage packet with a proper nebula header for our tunnel")
// this should log a statement and get ignored, allowing the real handshake packet to complete the tunnel
badPacket := stage1Packet.Copy()
badPacket.Data = badPacket.Data[:len(badPacket.Data)-nebula.HeaderLen]
myControl.InjectUDPPacket(badPacket)
// Have me consume their real stage 1 packet. I have a tunnel now
t.Log("Have me consume their real stage 1 packet. I have a tunnel now")
myControl.InjectUDPPacket(stage1Packet)
// Wait until we see my cached packet come through
t.Log("Wait until we see my cached packet come through")
myControl.WaitForType(1, 0, theirControl)
// Make sure our host infos are correct
t.Log("Make sure our host infos are correct")
assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIp, theirVpnIp, myControl, theirControl)
// Get that cached packet and make sure it looks right
t.Log("Get that cached packet and make sure it looks right")
myCachedPacket := theirControl.GetFromTun(true)
assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIp, theirVpnIp, 80, 80)
// Do a bidirectional tunnel test
t.Log("Do a bidirectional tunnel test")
assertTunnel(t, myVpnIp, theirVpnIp, myControl, theirControl, router.NewR(myControl, theirControl))
myControl.Stop()
@ -62,14 +63,17 @@ func TestGoodHandshake(t *testing.T) {
func TestWrongResponderHandshake(t *testing.T) {
ca, _, caKey, _ := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 1})
theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2})
evilControl, evilVpnIp, evilUdpAddr := newSimpleServer(ca, caKey, "evil", net.IP{10, 0, 0, 99})
// The IPs here are chosen on purpose:
// The current remote handling will sort by preference, public, and then lexically.
// So we need them to have a higher address than evil (we could apply a preference though)
myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 100})
theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 99})
evilControl, evilVpnIp, evilUdpAddr := newSimpleServer(ca, caKey, "evil", net.IP{10, 0, 0, 2})
// Add their real udp addr, which should be tried after evil. Doing this first because learned addresses are prepended
// Add their real udp addr, which should be tried after evil.
myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
// Put the evil udp addr in for their vpn Ip, this is a case of being lied to by the lighthouse. This will now be the first attempted ip
// Put the evil udp addr in for their vpn Ip, this is a case of being lied to by the lighthouse.
myControl.InjectLightHouseAddr(theirVpnIp, evilUdpAddr)
// Build a router so we don't have to reason who gets which packet
@ -80,137 +84,98 @@ func TestWrongResponderHandshake(t *testing.T) {
theirControl.Start()
evilControl.Start()
t.Log("Stand up the tunnel with evil (because the lighthouse cache is lying to us about who it is)")
t.Log("Start the handshake process, we will route until we see our cached packet get sent to them")
myControl.InjectTunUDPPacket(theirVpnIp, 80, 80, []byte("Hi from me"))
r.OnceFrom(myControl)
r.OnceFrom(evilControl)
r.RouteForAllExitFunc(func(p *nebula.UdpPacket, c *nebula.Control) router.ExitType {
h := &nebula.Header{}
err := h.Parse(p.Data)
if err != nil {
panic(err)
}
t.Log("I should have a tunnel with evil now and there should not be a cached packet waiting for us")
assertTunnel(t, myVpnIp, evilVpnIp, myControl, evilControl, r)
assertHostInfoPair(t, myUdpAddr, evilUdpAddr, myVpnIp, evilVpnIp, myControl, evilControl)
if p.ToIp.Equal(theirUdpAddr.IP) && p.ToPort == uint16(theirUdpAddr.Port) && h.Type == 1 {
return router.RouteAndExit
}
return router.KeepRouting
})
//TODO: Assert pending hostmap - I should have a correct hostinfo for them now
t.Log("Lets let the messages fly, this time we should have a tunnel with them")
r.OnceFrom(myControl)
r.OnceFrom(theirControl)
t.Log("I should now have a tunnel with them now and my original packet should get there")
r.RouteUntilAfterMsgType(myControl, 1, 0)
t.Log("My cached packet should be received by them")
myCachedPacket := theirControl.GetFromTun(true)
assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIp, theirVpnIp, 80, 80)
t.Log("I should now have a proper tunnel with them")
t.Log("Test the tunnel with them")
assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIp, theirVpnIp, myControl, theirControl)
assertTunnel(t, myVpnIp, theirVpnIp, myControl, theirControl, r)
t.Log("Lets make sure evil is still good")
assertTunnel(t, myVpnIp, evilVpnIp, myControl, evilControl, r)
t.Log("Flush all packets from all controllers")
r.FlushAll()
t.Log("Ensure ensure I don't have any hostinfo artifacts from evil")
assert.Nil(t, myControl.GetHostInfoByVpnIP(ip2int(evilVpnIp), true), "My pending hostmap should not contain evil")
assert.Nil(t, myControl.GetHostInfoByVpnIP(ip2int(evilVpnIp), false), "My main hostmap should not contain evil")
//NOTE: if evil lost the handshake race it may still have a tunnel since me would reject the handshake since the tunnel is complete
//TODO: assert hostmaps for everyone
t.Log("Success!")
//TODO: myControl is attempting to shut down 2 tunnels but is blocked on the udp txChan after the first close message
// what we really need here is a way to exit all the go routines loops (there are many)
//myControl.Stop()
//theirControl.Stop()
myControl.Stop()
theirControl.Stop()
}
////TODO: We need to test lies both as the race winner and race loser
//func TestManyWrongResponderHandshake(t *testing.T) {
// ca, _, caKey, _ := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
//
// myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 99})
// theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2})
// evilControl, evilVpnIp, evilUdpAddr := newSimpleServer(ca, caKey, "evil", net.IP{10, 0, 0, 1})
//
// t.Log("Build a router so we don't have to reason who gets which packet")
// r := newRouter(myControl, theirControl, evilControl)
//
// t.Log("Lets add more than 10 evil addresses, this exceeds the hostinfo remotes limit")
// for i := 0; i < 10; i++ {
// addr := net.UDPAddr{IP: evilUdpAddr.IP, Port: evilUdpAddr.Port + i}
// myControl.InjectLightHouseAddr(theirVpnIp, &addr)
// // We also need to tell our router about it
// r.AddRoute(addr.IP, uint16(addr.Port), evilControl)
// }
//
// // Start the servers
// myControl.Start()
// theirControl.Start()
// evilControl.Start()
//
// t.Log("Stand up the tunnel with evil (because the lighthouse cache is lying to us about who it is)")
// myControl.InjectTunUDPPacket(theirVpnIp, 80, 80, []byte("Hi from me"))
//
// t.Log("We need to spin until we get to the right remote for them")
// getOut := false
// injected := false
// for {
// t.Log("Routing for me and evil while we work through the bad ips")
// r.RouteExitFunc(myControl, func(packet *nebula.UdpPacket, receiver *nebula.Control) exitType {
// // We should stop routing right after we see a packet coming from us to them
// if *receiver == *theirControl {
// getOut = true
// return drainAndExit
// }
//
// // We need to poke our real ip in at some point, this is a well protected check looking for that moment
// if *receiver == *evilControl {
// hi := myControl.GetHostInfoByVpnIP(ip2int(theirVpnIp), true)
// if !injected && len(hi.RemoteAddrs) == 1 {
// t.Log("I am on my last ip for them, time to inject the real one into my lighthouse")
// myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
// injected = true
// }
// return drainAndExit
// }
//
// return keepRouting
// })
//
// if getOut {
// break
// }
//
// r.RouteForUntilAfterToAddr(evilControl, myUdpAddr, drainAndExit)
// }
//
// t.Log("I should have a tunnel with evil and them, evil should not have a cached packet")
// assertTunnel(t, myVpnIp, evilVpnIp, myControl, evilControl, r)
// evilHostInfo := myControl.GetHostInfoByVpnIP(ip2int(evilVpnIp), false)
// realEvilUdpAddr := &net.UDPAddr{IP: evilHostInfo.CurrentRemote.IP, Port: int(evilHostInfo.CurrentRemote.Port)}
//
// t.Log("Assert mine and evil's host pairs", evilUdpAddr, realEvilUdpAddr)
// assertHostInfoPair(t, myUdpAddr, realEvilUdpAddr, myVpnIp, evilVpnIp, myControl, evilControl)
//
// //t.Log("Draining everyones packets")
// //r.Drain(theirControl)
// //r.DrainAll(myControl, theirControl, evilControl)
// //
// //go func() {
// // for {
// // time.Sleep(10 * time.Millisecond)
// // t.Log(len(theirControl.GetUDPTxChan()))
// // t.Log(len(theirControl.GetTunTxChan()))
// // t.Log(len(myControl.GetUDPTxChan()))
// // t.Log(len(evilControl.GetUDPTxChan()))
// // t.Log("=====")
// // }
// //}()
//
// t.Log("I should have a tunnel with them now and my original packet should get there")
// r.RouteUntilAfterMsgType(myControl, 1, 0)
// myCachedPacket := theirControl.GetFromTun(true)
//
// t.Log("Got the cached packet, lets test the tunnel")
// assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIp, theirVpnIp, 80, 80)
//
// t.Log("Testing tunnels with them")
// assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIp, theirVpnIp, myControl, theirControl)
// assertTunnel(t, myVpnIp, theirVpnIp, myControl, theirControl, r)
//
// t.Log("Testing tunnels with evil")
// assertTunnel(t, myVpnIp, evilVpnIp, myControl, evilControl, r)
//
// //TODO: assert hostmaps for everyone
//}
func Test_Case1_Stage1Race(t *testing.T) {
ca, _, caKey, _ := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{})
myControl, myVpnIp, myUdpAddr := newSimpleServer(ca, caKey, "me ", net.IP{10, 0, 0, 1})
theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2})
// Put their info in our lighthouse and vice versa
myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
theirControl.InjectLightHouseAddr(myVpnIp, myUdpAddr)
// Build a router so we don't have to reason who gets which packet
r := router.NewR(myControl, theirControl)
// Start the servers
myControl.Start()
theirControl.Start()
t.Log("Trigger a handshake to start on both me and them")
myControl.InjectTunUDPPacket(theirVpnIp, 80, 80, []byte("Hi from me"))
theirControl.InjectTunUDPPacket(myVpnIp, 80, 80, []byte("Hi from them"))
t.Log("Get both stage 1 handshake packets")
myHsForThem := myControl.GetFromUDP(true)
theirHsForMe := theirControl.GetFromUDP(true)
t.Log("Now inject both stage 1 handshake packets")
myControl.InjectUDPPacket(theirHsForMe)
theirControl.InjectUDPPacket(myHsForThem)
//TODO: they should win, grab their index for me and make sure I use it in the end.
t.Log("They should not have a stage 2 (won the race) but I should send one")
theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
t.Log("Route for me until I send a message packet to them")
myControl.WaitForType(1, 0, theirControl)
t.Log("My cached packet should be received by them")
myCachedPacket := theirControl.GetFromTun(true)
assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIp, theirVpnIp, 80, 80)
t.Log("Route for them until I send a message packet to me")
theirControl.WaitForType(1, 0, myControl)
t.Log("Their cached packet should be received by me")
theirCachedPacket := myControl.GetFromTun(true)
assertUdpPacket(t, []byte("Hi from them"), theirCachedPacket, theirVpnIp, myVpnIp, 80, 80)
t.Log("Do a bidirectional tunnel test")
assertTunnel(t, myVpnIp, theirVpnIp, myControl, theirControl, r)
myControl.Stop()
theirControl.Stop()
//TODO: assert hostmaps
}
//TODO: add a test with many lies

View File

@ -64,6 +64,9 @@ func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, u
"host": "any",
}},
},
//"handshakes": m{
// "try_interval": "1s",
//},
"listen": m{
"host": udpAddr.IP.String(),
"port": udpAddr.Port,

3
e2e/router/doc.go Normal file
View File

@ -0,0 +1,3 @@
package router
// This file exists to allow `go fmt` to traverse here on its own. The build tags were keeping it out before

View File

@ -5,6 +5,7 @@ package router
import (
"fmt"
"net"
"reflect"
"strconv"
"sync"
@ -28,18 +29,18 @@ type R struct {
sync.Mutex
}
type exitType int
type ExitType int
const (
// Keeps routing, the function will get called again on the next packet
keepRouting exitType = 0
KeepRouting ExitType = 0
// Does not route this packet and exits immediately
exitNow exitType = 1
ExitNow ExitType = 1
// Routes this packet and exits immediately afterwards
routeAndExit exitType = 2
RouteAndExit ExitType = 2
)
type ExitFunc func(packet *nebula.UdpPacket, receiver *nebula.Control) exitType
type ExitFunc func(packet *nebula.UdpPacket, receiver *nebula.Control) ExitType
func NewR(controls ...*nebula.Control) *R {
r := &R{
@ -77,8 +78,8 @@ func (r *R) AddRoute(ip net.IP, port uint16, c *nebula.Control) {
// OnceFrom will route a single packet from sender then return
// If the router doesn't have the nebula controller for that address, we panic
func (r *R) OnceFrom(sender *nebula.Control) {
r.RouteExitFunc(sender, func(*nebula.UdpPacket, *nebula.Control) exitType {
return routeAndExit
r.RouteExitFunc(sender, func(*nebula.UdpPacket, *nebula.Control) ExitType {
return RouteAndExit
})
}
@ -116,7 +117,6 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []
// - exitNow: the packet will not be routed and this call will return immediately
// - routeAndExit: this call will return immediately after routing the last packet from sender
// - keepRouting: the packet will be routed and whatDo will be called again on the next packet from sender
//TODO: is this RouteWhile?
func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
h := &nebula.Header{}
for {
@ -136,16 +136,16 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
e := whatDo(p, receiver)
switch e {
case exitNow:
case ExitNow:
r.Unlock()
return
case routeAndExit:
case RouteAndExit:
receiver.InjectUDPPacket(p)
r.Unlock()
return
case keepRouting:
case KeepRouting:
receiver.InjectUDPPacket(p)
default:
@ -160,35 +160,135 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
// If the router doesn't have the nebula controller for that address, we panic
func (r *R) RouteUntilAfterMsgType(sender *nebula.Control, msgType nebula.NebulaMessageType, subType nebula.NebulaMessageSubType) {
h := &nebula.Header{}
r.RouteExitFunc(sender, func(p *nebula.UdpPacket, r *nebula.Control) exitType {
r.RouteExitFunc(sender, func(p *nebula.UdpPacket, r *nebula.Control) ExitType {
if err := h.Parse(p.Data); err != nil {
panic(err)
}
if h.Type == msgType && h.Subtype == subType {
return routeAndExit
return RouteAndExit
}
return keepRouting
return KeepRouting
})
}
// RouteForUntilAfterToAddr will route for sender and return only after it sees and sends a packet destined for toAddr
// finish can be any of the exitType values except `keepRouting`, the default value is `routeAndExit`
// If the router doesn't have the nebula controller for that address, we panic
func (r *R) RouteForUntilAfterToAddr(sender *nebula.Control, toAddr *net.UDPAddr, finish exitType) {
if finish == keepRouting {
finish = routeAndExit
func (r *R) RouteForUntilAfterToAddr(sender *nebula.Control, toAddr *net.UDPAddr, finish ExitType) {
if finish == KeepRouting {
finish = RouteAndExit
}
r.RouteExitFunc(sender, func(p *nebula.UdpPacket, r *nebula.Control) exitType {
r.RouteExitFunc(sender, func(p *nebula.UdpPacket, r *nebula.Control) ExitType {
if p.ToIp.Equal(toAddr.IP) && p.ToPort == uint16(toAddr.Port) {
return finish
}
return keepRouting
return KeepRouting
})
}
// RouteForAllExitFunc will route for every registered controller and calls the whatDo func with each udp packet from
// whatDo can return:
// - exitNow: the packet will not be routed and this call will return immediately
// - routeAndExit: this call will return immediately after routing the last packet from sender
// - keepRouting: the packet will be routed and whatDo will be called again on the next packet from sender
func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
sc := make([]reflect.SelectCase, len(r.controls))
cm := make([]*nebula.Control, len(r.controls))
i := 0
for _, c := range r.controls {
sc[i] = reflect.SelectCase{
Dir: reflect.SelectRecv,
Chan: reflect.ValueOf(c.GetUDPTxChan()),
Send: reflect.Value{},
}
cm[i] = c
i++
}
for {
x, rx, _ := reflect.Select(sc)
r.Lock()
p := rx.Interface().(*nebula.UdpPacket)
outAddr := cm[x].GetUDPAddr()
inAddr := net.JoinHostPort(p.ToIp.String(), fmt.Sprintf("%v", p.ToPort))
receiver := r.getControl(outAddr, inAddr, p)
if receiver == nil {
r.Unlock()
panic("Can't route for host: " + inAddr)
}
e := whatDo(p, receiver)
switch e {
case ExitNow:
r.Unlock()
return
case RouteAndExit:
receiver.InjectUDPPacket(p)
r.Unlock()
return
case KeepRouting:
receiver.InjectUDPPacket(p)
default:
panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
}
r.Unlock()
}
}
// FlushAll will route for every registered controller, exiting once there are no packets left to route
func (r *R) FlushAll() {
sc := make([]reflect.SelectCase, len(r.controls))
cm := make([]*nebula.Control, len(r.controls))
i := 0
for _, c := range r.controls {
sc[i] = reflect.SelectCase{
Dir: reflect.SelectRecv,
Chan: reflect.ValueOf(c.GetUDPTxChan()),
Send: reflect.Value{},
}
cm[i] = c
i++
}
// Add a default case to exit when nothing is left to send
sc = append(sc, reflect.SelectCase{
Dir: reflect.SelectDefault,
Chan: reflect.Value{},
Send: reflect.Value{},
})
for {
x, rx, ok := reflect.Select(sc)
if !ok {
return
}
r.Lock()
p := rx.Interface().(*nebula.UdpPacket)
outAddr := cm[x].GetUDPAddr()
inAddr := net.JoinHostPort(p.ToIp.String(), fmt.Sprintf("%v", p.ToPort))
receiver := r.getControl(outAddr, inAddr, p)
if receiver == nil {
r.Unlock()
panic("Can't route for host: " + inAddr)
}
r.Unlock()
}
}
// getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change
// This is an internal router function, the caller must hold the lock
func (r *R) getControl(fromAddr, toAddr string, p *nebula.UdpPacket) *nebula.Control {
@ -216,6 +316,5 @@ func (r *R) getControl(fromAddr, toAddr string, p *nebula.UdpPacket) *nebula.Con
return c
}
//TODO: call receive hooks!
return r.controls[toAddr]
}