Lighthouse performance pass (#418)

Makefile

@@ -126,9 +126,9 @@ bench-cpu-long:
 proto: nebula.pb.go cert/cert.pb.go
 
 nebula.pb.go: nebula.proto .FORCE
-	go build google.golang.org/protobuf/cmd/protoc-gen-go
+	go build github.com/gogo/protobuf/protoc-gen-gogofaster
-	PATH="$(CURDIR):$(PATH)" protoc --go_out=. --go_opt=paths=source_relative $<
+	PATH="$(CURDIR):$(PATH)" protoc --gogofaster_out=. $<
-	rm protoc-gen-go
+	rm protoc-gen-gogofaster
 
 cert/cert.pb.go: cert/cert.proto .FORCE
 	$(MAKE) -C cert cert.pb.go

e2e/handshakes_test.go

@@ -66,12 +66,12 @@ func TestWrongResponderHandshake(t *testing.T) {
 	theirControl, theirVpnIp, theirUdpAddr := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2})
 	evilControl, evilVpnIp, evilUdpAddr := newSimpleServer(ca, caKey, "evil", net.IP{10, 0, 0, 99})
 
-	// Put the evil udp addr in for their vpn Ip, this is a case of being lied to by the lighthouse
+	// Add their real udp addr, which should be tried after evil. Doing this first because learned addresses are prepended
-	myControl.InjectLightHouseAddr(theirVpnIp, evilUdpAddr)
-
-	// But also add their real udp addr, which should be tried after evil
 	myControl.InjectLightHouseAddr(theirVpnIp, theirUdpAddr)
 
+	// Put the evil udp addr in for their vpn Ip, this is a case of being lied to by the lighthouse. This will now be the first attempted ip
+	myControl.InjectLightHouseAddr(theirVpnIp, evilUdpAddr)
+
 	// Build a router so we don't have to reason who gets which packet
 	r := router.NewR(myControl, theirControl, evilControl)
 

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
 	github.com/flynn/noise v0.0.0-20210331153838-4bdb43be3117
+	github.com/gogo/protobuf v1.3.2
 	github.com/golang/protobuf v1.5.0
 	github.com/google/gopacket v1.1.19
 	github.com/imdario/mergo v0.3.8

go.sum

@@ -28,6 +28,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -47,6 +49,8 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kardianos/service v1.1.0 h1:QV2SiEeWK42P0aEmGcsAgjApw/lRxkwopvT+Gu6t1/0=
 github.com/kardianos/service v1.1.0/go.mod h1:RrJI2xn5vve/r32U5suTbeaSGoMU6GbNPoj36CVYcHc=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -107,26 +111,34 @@ github.com/vishvananda/netlink v1.0.1-0.20190522153524-00009fb8606a h1:Bt1IVPhiC
 github.com/vishvananda/netlink v1.0.1-0.20190522153524-00009fb8606a/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -136,6 +148,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
@@ -145,11 +158,15 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=

handshake_manager_test.go

@@ -142,8 +142,11 @@ func Test_NewHandshakeManagerTrigger(t *testing.T) {
 	hi := blah.pendingHostMap.Hosts[ip]
 	assert.Nil(t, hi.remote)
 
-	lh.addrMap = map[uint32][]*udpAddr{
+	uaddr := NewUDPAddrFromString("10.1.1.1:4242")
-		ip: {NewUDPAddrFromString("10.1.1.1:4242")},
+	lh.addrMap = map[uint32]*ip4And6{}
+	lh.addrMap[ip] = &ip4And6{
+		v4: []*Ip4AndPort{NewIp4AndPort(uaddr.IP, uint32(uaddr.Port))},
+		v6: []*Ip6AndPort{},
 	}
 
 	// This should trigger the hostmap to populate the hostinfo

inside.go

@@ -247,7 +247,7 @@ func (f *Interface) sendNoMetrics(t NebulaMessageType, st NebulaMessageSubType,
 	if hostinfo.lastRebindCount != f.rebindCount {
 		//NOTE: there is an update hole if a tunnel isn't used and exactly 256 rebinds occur before the tunnel is
 		// finally used again. This tunnel would eventually be torn down and recreated if this action didn't help.
-		f.lightHouse.Query(hostinfo.hostId, f)
+		f.lightHouse.QueryServer(hostinfo.hostId, f)
 		hostinfo.lastRebindCount = f.rebindCount
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("vpnIp", hostinfo.hostId).Debug("Lighthouse update triggered for punch due to rebind counter")

lighthouse.go

@@ -1,6 +1,7 @@
 package nebula
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"net"
@@ -10,19 +11,38 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
-	"github.com/slackhq/nebula/cert"
 )
 
+//TODO: if the pb code for ipv6 used a fixed data type we could save more work
+//TODO: nodes are roaming lighthouses, this is bad. How are they learning?
+//TODO: as a lh client, ignore any address within my nebula network?????
+
 var ErrHostNotKnown = errors.New("host not known")
 
+// The maximum number of ip addresses to store for a given vpnIp per address family
+const maxAddrs = 10
+
+type ip4And6 struct {
+	//TODO: adding a lock here could allow us to release the lock on lh.addrMap quicker
+
+	// v4 and v6 store addresses that have been self reported by the client
+	v4 []*Ip4AndPort
+	v6 []*Ip6AndPort
+
+	// Learned addresses are ones that a client does not know about but a lighthouse learned from as a result of the received packet
+	learnedV4 []*Ip4AndPort
+	learnedV6 []*Ip6AndPort
+}
+
 type LightHouse struct {
+	//TODO: We need a timer wheel to kick out vpnIps that haven't reported in a long time
 	sync.RWMutex //Because we concurrently read and write to our maps
 	amLighthouse bool
 	myIp         uint32
 	punchConn    *udpConn
 
 	// Local cache of answers from light houses
-	addrMap map[uint32][]*udpAddr
+	addrMap map[uint32]*ip4And6
 
 	// filters remote addresses allowed for each host
 	// - When we are a lighthouse, this filters what addresses we store and
@@ -53,14 +73,13 @@ type LightHouse struct {
 
 type EncWriter interface {
 	SendMessageToVpnIp(t NebulaMessageType, st NebulaMessageSubType, vpnIp uint32, p, nb, out []byte)
-	SendMessageToAll(t NebulaMessageType, st NebulaMessageSubType, vpnIp uint32, p, nb, out []byte)
 }
 
 func NewLightHouse(l *logrus.Logger, amLighthouse bool, myIp uint32, ips []uint32, interval int, nebulaPort uint32, pc *udpConn, punchBack bool, punchDelay time.Duration, metricsEnabled bool) *LightHouse {
 	h := LightHouse{
 		amLighthouse: amLighthouse,
 		myIp:         myIp,
-		addrMap:      make(map[uint32][]*udpAddr),
+		addrMap:      make(map[uint32]*ip4And6),
 		nebulaPort:   nebulaPort,
 		lighthouses:  make(map[uint32]struct{}),
 		staticList:   make(map[uint32]struct{}),
@@ -110,13 +129,14 @@ func (lh *LightHouse) ValidateLHStaticEntries() error {
 }
 
 func (lh *LightHouse) Query(ip uint32, f EncWriter) ([]*udpAddr, error) {
+	//TODO: we need to hold the lock through the next func
 	if !lh.IsLighthouseIP(ip) {
 		lh.QueryServer(ip, f)
 	}
 	lh.RLock()
 	if v, ok := lh.addrMap[ip]; ok {
 		lh.RUnlock()
-		return v, nil
+		return TransformLHReplyToUdpAddrs(v), nil
 	}
 	lh.RUnlock()
 	return nil, ErrHostNotKnown
@@ -141,17 +161,29 @@ func (lh *LightHouse) QueryServer(ip uint32, f EncWriter) {
 	}
 }
 
-// Query our local lighthouse cached results
 func (lh *LightHouse) QueryCache(ip uint32) []*udpAddr {
+	//TODO: we need to hold the lock through the next func
 	lh.RLock()
 	if v, ok := lh.addrMap[ip]; ok {
 		lh.RUnlock()
-		return v
+		return TransformLHReplyToUdpAddrs(v)
 	}
 	lh.RUnlock()
 	return nil
 }
 
+//
+func (lh *LightHouse) queryAndPrepMessage(ip uint32, f func(*ip4And6) (int, error)) (bool, int, error) {
+	lh.RLock()
+	if v, ok := lh.addrMap[ip]; ok {
+		n, err := f(v)
+		lh.RUnlock()
+		return true, n, err
+	}
+	lh.RUnlock()
+	return false, 0, nil
+}
+
 func (lh *LightHouse) DeleteVpnIP(vpnIP uint32) {
 	// First we check the static mapping
 	// and do nothing if it is there
@@ -161,11 +193,46 @@ func (lh *LightHouse) DeleteVpnIP(vpnIP uint32) {
 	lh.Lock()
 	//l.Debugln(lh.addrMap)
 	delete(lh.addrMap, vpnIP)
+
+	if lh.l.Level >= logrus.DebugLevel {
 		lh.l.Debugf("deleting %s from lighthouse.", IntIp(vpnIP))
+	}
+
 	lh.Unlock()
 }
 
-func (lh *LightHouse) AddRemote(vpnIP uint32, toIp *udpAddr, static bool) {
+// AddRemote is correct way for non LightHouse members to add an address. toAddr will be placed in the learned map
+// static means this is a static host entry from the config file, it should only be used on start up
+func (lh *LightHouse) AddRemote(vpnIP uint32, toAddr *udpAddr, static bool) {
+	if ipv4 := toAddr.IP.To4(); ipv4 != nil {
+		lh.addRemoteV4(vpnIP, NewIp4AndPort(ipv4, uint32(toAddr.Port)), static, true)
+	} else {
+		lh.addRemoteV6(vpnIP, NewIp6AndPort(toAddr.IP, uint32(toAddr.Port)), static, true)
+	}
+
+	//TODO: if we do not add due to a config filter we may end up not having any addresses here
+	if static {
+		lh.staticList[vpnIP] = struct{}{}
+	}
+}
+
+// unsafeGetAddrs assumes you have the lh lock
+func (lh *LightHouse) unsafeGetAddrs(vpnIP uint32) *ip4And6 {
+	am, ok := lh.addrMap[vpnIP]
+	if !ok {
+		am = &ip4And6{
+			v4:        make([]*Ip4AndPort, 0),
+			v6:        make([]*Ip6AndPort, 0),
+			learnedV4: make([]*Ip4AndPort, 0),
+			learnedV6: make([]*Ip6AndPort, 0),
+		}
+		lh.addrMap[vpnIP] = am
+	}
+	return am
+}
+
+// addRemoteV4 is a lighthouse internal method that prepends a remote if it is allowed by the allow list and not duplicated
+func (lh *LightHouse) addRemoteV4(vpnIP uint32, to *Ip4AndPort, static bool, learned bool) {
 	// First we check if the sender thinks this is a static entry
 	// and do nothing if it is not, but should be considered static
 	if static == false {
@@ -176,24 +243,108 @@ func (lh *LightHouse) AddRemote(vpnIP uint32, toIp *udpAddr, static bool) {
 
 	lh.Lock()
 	defer lh.Unlock()
-	for _, v := range lh.addrMap[vpnIP] {
+	am := lh.unsafeGetAddrs(vpnIP)
-		if v.Equals(toIp) {
+
+	if learned {
+		if !lh.unlockedShouldAddV4(am.learnedV4, to) {
 			return
 		}
+		am.learnedV4 = prependAndLimitV4(am.learnedV4, to)
+	} else {
+		if !lh.unlockedShouldAddV4(am.v4, to) {
+			return
+		}
+		am.v4 = prependAndLimitV4(am.v4, to)
+	}
+}
+
+func prependAndLimitV4(cache []*Ip4AndPort, to *Ip4AndPort) []*Ip4AndPort {
+	cache = append(cache, nil)
+	copy(cache[1:], cache)
+	cache[0] = to
+	if len(cache) > MaxRemotes {
+		cache = cache[:maxAddrs]
+	}
+	return cache
+}
+
+// unlockedShouldAddV4 checks if to is allowed by our allow list and is not already present in the cache
+func (lh *LightHouse) unlockedShouldAddV4(am []*Ip4AndPort, to *Ip4AndPort) bool {
+	ip := int2ip(to.Ip)
+	allow := lh.remoteAllowList.Allow(ip)
+	if lh.l.Level >= logrus.DebugLevel {
+		lh.l.WithField("remoteIp", ip).WithField("allow", allow).Debug("remoteAllowList.Allow")
 	}
 
-	allow := lh.remoteAllowList.Allow(toIp.IP)
-	lh.l.WithField("remoteIp", toIp).WithField("allow", allow).Debug("remoteAllowList.Allow")
 	if !allow {
+		return false
+	}
+
+	for _, v := range am {
+		if v.Ip == to.Ip && v.Port == to.Port {
+			return false
+		}
+	}
+
+	return true
+}
+
+// addRemoteV6 is a lighthouse internal method that prepends a remote if it is allowed by the allow list and not duplicated
+func (lh *LightHouse) addRemoteV6(vpnIP uint32, to *Ip6AndPort, static bool, learned bool) {
+	// First we check if the sender thinks this is a static entry
+	// and do nothing if it is not, but should be considered static
+	if static == false {
+		if _, ok := lh.staticList[vpnIP]; ok {
 			return
 		}
-
-	//l.Debugf("Adding reply of %s as %s\n", IntIp(vpnIP), toIp)
-	if static {
-		lh.staticList[vpnIP] = struct{}{}
 	}
 
-	lh.addrMap[vpnIP] = append(lh.addrMap[vpnIP], toIp.Copy())
+	lh.Lock()
+	defer lh.Unlock()
+	am := lh.unsafeGetAddrs(vpnIP)
+
+	if learned {
+		if !lh.unlockedShouldAddV6(am.learnedV6, to) {
+			return
+		}
+		am.learnedV6 = prependAndLimitV6(am.learnedV6, to)
+	} else {
+		if !lh.unlockedShouldAddV6(am.v6, to) {
+			return
+		}
+		am.v6 = prependAndLimitV6(am.v6, to)
+	}
+}
+
+func prependAndLimitV6(cache []*Ip6AndPort, to *Ip6AndPort) []*Ip6AndPort {
+	cache = append(cache, nil)
+	copy(cache[1:], cache)
+	cache[0] = to
+	if len(cache) > MaxRemotes {
+		cache = cache[:maxAddrs]
+	}
+	return cache
+}
+
+// unlockedShouldAddV6 checks if to is allowed by our allow list and is not already present in the cache
+func (lh *LightHouse) unlockedShouldAddV6(am []*Ip6AndPort, to *Ip6AndPort) bool {
+	ip := net.IP(to.Ip)
+	allow := lh.remoteAllowList.Allow(ip)
+	if lh.l.Level >= logrus.DebugLevel {
+		lh.l.WithField("remoteIp", ip).WithField("allow", allow).Debug("remoteAllowList.Allow")
+	}
+
+	if !allow {
+		return false
+	}
+
+	for _, v := range am {
+		if bytes.Equal(v.Ip, to.Ip) && v.Port == to.Port {
+			return false
+		}
+	}
+
+	return true
 }
 
 func (lh *LightHouse) AddRemoteAndReset(vpnIP uint32, toIp *udpAddr) {
@@ -201,7 +352,6 @@ func (lh *LightHouse) AddRemoteAndReset(vpnIP uint32, toIp *udpAddr) {
 		lh.DeleteVpnIP(vpnIP)
 		lh.AddRemote(vpnIP, toIp, false)
 	}
-
 }
 
 func (lh *LightHouse) IsLighthouseIP(vpnIP uint32) bool {
@@ -220,32 +370,20 @@ func NewLhQueryByInt(VpnIp uint32) *NebulaMeta {
 	}
 }
 
-type ip4Or6 struct {
+func NewIp4AndPort(ip net.IP, port uint32) *Ip4AndPort {
-	v4 IpAndPort
+	ipp := Ip4AndPort{Port: port}
-	v6 Ip6AndPort
+	ipp.Ip = ip2int(ip)
+	return &ipp
 }
 
-func NewIpAndPort(ip net.IP, port uint32) ip4Or6 {
+func NewIp6AndPort(ip net.IP, port uint32) *Ip6AndPort {
-	ipp := ip4Or6{}
+	ipp := Ip6AndPort{Port: port}
-
+	ipp.Ip = make([]byte, len(ip))
-	if ipv4 := ip.To4(); ipv4 != nil {
+	copy(ipp.Ip, ip)
-		ipp.v4 = IpAndPort{Port: port}
+	return &ipp
-		ipp.v4.Ip = ip2int(ip)
-
-	} else {
-		ipp.v6 = Ip6AndPort{Port: port}
-		ipp.v6.Ip = make([]byte, len(ip))
-		copy(ipp.v6.Ip, ip)
 }
 
-	return ipp
+func NewUDPAddrFromLH4(ipp *Ip4AndPort) *udpAddr {
-}
-
-func NewIpAndPortFromUDPAddr(addr *udpAddr) ip4Or6 {
-	return NewIpAndPort(addr.IP, uint32(addr.Port))
-}
-
-func NewUDPAddrFromLH4(ipp *IpAndPort) *udpAddr {
 	ip := ipp.Ip
 	return NewUDPAddr(
 		net.IPv4(byte(ip&0xff000000>>24), byte(ip&0x00ff0000>>16), byte(ip&0x0000ff00>>8), byte(ip&0x000000ff)),
@@ -269,26 +407,26 @@ func (lh *LightHouse) LhUpdateWorker(f EncWriter) {
 }
 
 func (lh *LightHouse) SendUpdate(f EncWriter) {
-	var v4 []*IpAndPort
+	var v4 []*Ip4AndPort
 	var v6 []*Ip6AndPort
 
 	for _, e := range *localIps(lh.l, lh.localAllowList) {
-		// Only add IPs that aren't my VPN/tun IP
+		if ip2int(e) == lh.myIp {
-		if ip2int(e) != lh.myIp {
+			continue
-			ipp := NewIpAndPort(e, lh.nebulaPort)
-			if len(ipp.v6.Ip) > 0 {
-				v6 = append(v6, &ipp.v6)
-			} else {
-				v4 = append(v4, &ipp.v4)
 		}
 
+		// Only add IPs that aren't my VPN/tun IP
+		if ip := e.To4(); ip != nil {
+			v4 = append(v4, NewIp4AndPort(e, lh.nebulaPort))
+		} else {
+			v6 = append(v6, NewIp6AndPort(e, lh.nebulaPort))
 		}
 	}
 	m := &NebulaMeta{
 		Type: NebulaMeta_HostUpdateNotification,
 		Details: &NebulaMetaDetails{
 			VpnIp:       lh.myIp,
-			IpAndPorts:  v4,
+			Ip4AndPorts: v4,
 			Ip6AndPorts: v6,
 		},
 	}
@@ -298,7 +436,7 @@ func (lh *LightHouse) SendUpdate(f EncWriter) {
 	out := make([]byte, mtu)
 	for vpnIp := range lh.lighthouses {
 		mm, err := proto.Marshal(m)
-		if err != nil {
+		if err != nil && lh.l.Level >= logrus.DebugLevel {
 			lh.l.Debugf("Invalid marshal to update")
 		}
 		//l.Error("LIGHTHOUSE PACKET SEND", mm)
@@ -311,9 +449,9 @@ type LightHouseHandler struct {
 	lh   *LightHouse
 	nb   []byte
 	out  []byte
+	pb   []byte
 	meta *NebulaMeta
-	iap  []ip4Or6
+	l    *logrus.Logger
-	iapp []*ip4Or6
 }
 
 func (lh *LightHouse) NewRequestHandler() *LightHouseHandler {
@@ -321,288 +459,283 @@ func (lh *LightHouse) NewRequestHandler() *LightHouseHandler {
 		lh:  lh,
 		nb:  make([]byte, 12, 12),
 		out: make([]byte, mtu),
+		l:   lh.l,
+		pb:  make([]byte, mtu),
 
 		meta: &NebulaMeta{
 			Details: &NebulaMetaDetails{},
 		},
 	}
 
-	lhh.resizeIpAndPorts(10)
-
 	return lhh
 }
 
+func (lh *LightHouse) metricRx(t NebulaMeta_MessageType, i int64) {
+	lh.metrics.Rx(NebulaMessageType(t), 0, i)
+}
+
+func (lh *LightHouse) metricTx(t NebulaMeta_MessageType, i int64) {
+	lh.metrics.Tx(NebulaMessageType(t), 0, i)
+}
+
 // This method is similar to Reset(), but it re-uses the pointer structs
 // so that we don't have to re-allocate them
 func (lhh *LightHouseHandler) resetMeta() *NebulaMeta {
 	details := lhh.meta.Details
-
-	details.Reset()
 	lhh.meta.Reset()
+
+	// Keep the array memory around
+	details.Ip4AndPorts = details.Ip4AndPorts[:0]
+	details.Ip6AndPorts = details.Ip6AndPorts[:0]
 	lhh.meta.Details = details
 
 	return lhh.meta
 }
 
-func (lhh *LightHouseHandler) resizeIpAndPorts(n int) {
+//TODO: do we need c here?
-	if cap(lhh.iap) < n {
+func (lhh *LightHouseHandler) HandleRequest(rAddr *udpAddr, vpnIp uint32, p []byte, w EncWriter) {
-		lhh.iap = make([]ip4Or6, n)
-		lhh.iapp = make([]*ip4Or6, n)
-
-		for i := range lhh.iap {
-			lhh.iapp[i] = &lhh.iap[i]
-		}
-	}
-	lhh.iap = lhh.iap[:n]
-	lhh.iapp = lhh.iapp[:n]
-}
-
-func (lhh *LightHouseHandler) setIpAndPortsFromNetIps(ips []*udpAddr) []*ip4Or6 {
-	lhh.resizeIpAndPorts(len(ips))
-	for i, e := range ips {
-		lhh.iap[i] = NewIpAndPortFromUDPAddr(e)
-	}
-	return lhh.iapp
-}
-
-func (lhh *LightHouseHandler) HandleRequest(rAddr *udpAddr, vpnIp uint32, p []byte, c *cert.NebulaCertificate, f EncWriter) {
-	lh := lhh.lh
 	n := lhh.resetMeta()
-	err := proto.UnmarshalMerge(p, n)
+	err := n.Unmarshal(p)
 	if err != nil {
-		lh.l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).WithField("udpAddr", rAddr).
+		lhh.l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).WithField("udpAddr", rAddr).
 			Error("Failed to unmarshal lighthouse packet")
 		//TODO: send recv_error?
 		return
 	}
 
 	if n.Details == nil {
-		lh.l.WithField("vpnIp", IntIp(vpnIp)).WithField("udpAddr", rAddr).
+		lhh.l.WithField("vpnIp", IntIp(vpnIp)).WithField("udpAddr", rAddr).
 			Error("Invalid lighthouse update")
 		//TODO: send recv_error?
 		return
 	}
 
-	lh.metricRx(n.Type, 1)
+	lhh.lh.metricRx(n.Type, 1)
 
 	switch n.Type {
 	case NebulaMeta_HostQuery:
+		lhh.handleHostQuery(n, vpnIp, rAddr, w)
+
+	case NebulaMeta_HostQueryReply:
+		lhh.handleHostQueryReply(n, vpnIp)
+
+	case NebulaMeta_HostUpdateNotification:
+		lhh.handleHostUpdateNotification(n, vpnIp)
+
+	case NebulaMeta_HostMovedNotification:
+	case NebulaMeta_HostPunchNotification:
+		lhh.handleHostPunchNotification(n, vpnIp, w)
+	}
+}
+
+func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, vpnIp uint32, addr *udpAddr, w EncWriter) {
 	// Exit if we don't answer queries
-		if !lh.amLighthouse {
+	if !lhh.lh.amLighthouse {
-			lh.l.Debugln("I don't answer queries, but received from: ", rAddr)
+		if lhh.l.Level >= logrus.DebugLevel {
+			lhh.l.Debugln("I don't answer queries, but received from: ", addr)
+		}
 		return
 	}
 
-		//l.Debugln("Got Query")
+	//TODO: we can DRY this further
-		ips, err := lh.Query(n.Details.VpnIp, f)
-		if err != nil {
-			//l.Debugf("Can't answer query %s from %s because error: %s", IntIp(n.Details.VpnIp), rAddr, err)
-			return
-		} else {
 	reqVpnIP := n.Details.VpnIp
+	//TODO: Maybe instead of marshalling into n we marshal into a new `r` to not nuke our current request data
+	//TODO: If we use a lock on cache we can avoid holding it on lh.addrMap and keep things moving better
+	found, ln, err := lhh.lh.queryAndPrepMessage(n.Details.VpnIp, func(cache *ip4And6) (int, error) {
 		n = lhh.resetMeta()
 		n.Type = NebulaMeta_HostQueryReply
 		n.Details.VpnIp = reqVpnIP
 
-			v4s := make([]*IpAndPort, 0)
+		lhh.coalesceAnswers(cache, n)
-			v6s := make([]*Ip6AndPort, 0)
-			for _, v := range lhh.setIpAndPortsFromNetIps(ips) {
-				if len(v.v6.Ip) > 0 {
-					v6s = append(v6s, &v.v6)
-				} else {
-					v4s = append(v4s, &v.v4)
-				}
-			}
 
-			if len(v4s) > 0 {
+		return n.MarshalTo(lhh.pb)
-				n.Details.IpAndPorts = v4s
+	})
-			}
 
-			if len(v6s) > 0 {
+	if !found {
-				n.Details.Ip6AndPorts = v6s
-			}
-
-			reply, err := proto.Marshal(n)
-			if err != nil {
-				lh.l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).Error("Failed to marshal lighthouse host query reply")
 		return
 	}
-			lh.metricTx(NebulaMeta_HostQueryReply, 1)
+
-			f.SendMessageToVpnIp(lightHouse, 0, vpnIp, reply, lhh.nb, lhh.out[:0])
+	if err != nil {
+		lhh.l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).Error("Failed to marshal lighthouse host query reply")
+		return
+	}
+
+	lhh.lh.metricTx(NebulaMeta_HostQueryReply, 1)
+	w.SendMessageToVpnIp(lightHouse, 0, vpnIp, lhh.pb[:ln], lhh.nb, lhh.out[:0])
 
 	// This signals the other side to punch some zero byte udp packets
-			ips, err = lh.Query(vpnIp, f)
+	found, ln, err = lhh.lh.queryAndPrepMessage(vpnIp, func(cache *ip4And6) (int, error) {
-			if err != nil {
-				lh.l.WithField("vpnIp", IntIp(vpnIp)).Debugln("Can't notify host to punch")
-				return
-			} else {
-				//l.Debugln("Notify host to punch", iap)
 		n = lhh.resetMeta()
 		n.Type = NebulaMeta_HostPunchNotification
 		n.Details.VpnIp = vpnIp
 
-				v4s := make([]*IpAndPort, 0)
+		lhh.coalesceAnswers(cache, n)
-				v6s := make([]*Ip6AndPort, 0)
-				for _, v := range lhh.setIpAndPortsFromNetIps(ips) {
-					if len(v.v6.Ip) > 0 {
-						v6s = append(v6s, &v.v6)
-					} else {
-						v4s = append(v4s, &v.v4)
-					}
-				}
 
-				if len(v4s) > 0 {
+		return n.MarshalTo(lhh.pb)
-					n.Details.IpAndPorts = v4s
+	})
-				}
 
-				if len(v6s) > 0 {
+	if !found {
-					n.Details.Ip6AndPorts = v6s
-				}
-
-				reply, _ := proto.Marshal(n)
-				lh.metricTx(NebulaMeta_HostPunchNotification, 1)
-				f.SendMessageToVpnIp(lightHouse, 0, reqVpnIP, reply, lhh.nb, lhh.out[:0])
-			}
-			//fmt.Println(reply, remoteaddr)
-		}
-
-	case NebulaMeta_HostQueryReply:
-		if !lh.IsLighthouseIP(vpnIp) {
 		return
 	}
 
-		for _, a := range n.Details.IpAndPorts {
+	if err != nil {
-			ans := NewUDPAddrFromLH4(a)
+		lhh.l.WithError(err).WithField("vpnIp", IntIp(vpnIp)).Error("Failed to marshal lighthouse host was queried for")
-			if ans != nil {
+		return
-				lh.AddRemote(n.Details.VpnIp, ans, false)
-			}
 	}
 
-		for _, a := range n.Details.Ip6AndPorts {
+	lhh.lh.metricTx(NebulaMeta_HostPunchNotification, 1)
-			ans := NewUDPAddrFromLH6(a)
+	w.SendMessageToVpnIp(lightHouse, 0, reqVpnIP, lhh.pb[:ln], lhh.nb, lhh.out[:0])
-			if ans != nil {
-				lh.AddRemote(n.Details.VpnIp, ans, false)
 }
+
+func (lhh *LightHouseHandler) coalesceAnswers(cache *ip4And6, n *NebulaMeta) {
+	n.Details.Ip4AndPorts = append(n.Details.Ip4AndPorts, cache.v4...)
+	n.Details.Ip4AndPorts = append(n.Details.Ip4AndPorts, cache.learnedV4...)
+
+	n.Details.Ip6AndPorts = append(n.Details.Ip6AndPorts, cache.v6...)
+	n.Details.Ip6AndPorts = append(n.Details.Ip6AndPorts, cache.learnedV6...)
+}
+
+func (lhh *LightHouseHandler) handleHostQueryReply(n *NebulaMeta, vpnIp uint32) {
+	if !lhh.lh.IsLighthouseIP(vpnIp) {
+		return
+	}
+
+	// We can't just slam the responses in as they may come from multiple lighthouses and we should coalesce the answers
+	for _, to := range n.Details.Ip4AndPorts {
+		lhh.lh.addRemoteV4(n.Details.VpnIp, to, false, false)
+	}
+
+	for _, to := range n.Details.Ip6AndPorts {
+		lhh.lh.addRemoteV6(n.Details.VpnIp, to, false, false)
 	}
 
 	// Non-blocking attempt to trigger, skip if it would block
 	select {
-		case lh.handshakeTrigger <- n.Details.VpnIp:
+	case lhh.lh.handshakeTrigger <- n.Details.VpnIp:
 	default:
 	}
+}
 
-	case NebulaMeta_HostUpdateNotification:
+func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, vpnIp uint32) {
-		//Simple check that the host sent this not someone else
+	if !lhh.lh.amLighthouse {
-		if n.Details.VpnIp != vpnIp {
+		if lhh.l.Level >= logrus.DebugLevel {
-			lh.l.WithField("vpnIp", IntIp(vpnIp)).WithField("answer", IntIp(n.Details.VpnIp)).Debugln("Host sent invalid update")
+			lhh.l.Debugln("I am not a lighthouse, do not take host updates: ", vpnIp)
+		}
 		return
 	}
 
-		for _, a := range n.Details.IpAndPorts {
+	//Simple check that the host sent this not someone else
-			ans := NewUDPAddrFromLH4(a)
+	if n.Details.VpnIp != vpnIp {
-			if ans != nil {
+		if lhh.l.Level >= logrus.DebugLevel {
-				lh.AddRemote(n.Details.VpnIp, ans, false)
+			lhh.l.WithField("vpnIp", IntIp(vpnIp)).WithField("answer", IntIp(n.Details.VpnIp)).Debugln("Host sent invalid update")
+		}
+		return
+	}
+
+	lhh.lh.Lock()
+	defer lhh.lh.Unlock()
+	am := lhh.lh.unsafeGetAddrs(vpnIp)
+
+	//TODO: other note on a lock for am so we can release more quickly and lock our real unit of change which is far less contended
+	//TODO: we are not filtering by local or remote allowed addrs here, is this an ok change to make?
+
+	// We don't accumulate addresses being told to us
+	am.v4 = am.v4[:0]
+	am.v6 = am.v6[:0]
+
+	for _, v := range n.Details.Ip4AndPorts {
+		if lhh.lh.unlockedShouldAddV4(am.v4, v) {
+			am.v4 = append(am.v4, v)
 		}
 	}
 
-		for _, a := range n.Details.Ip6AndPorts {
+	for _, v := range n.Details.Ip6AndPorts {
-			ans := NewUDPAddrFromLH6(a)
+		if lhh.lh.unlockedShouldAddV6(am.v6, v) {
-			if ans != nil {
+			am.v6 = append(am.v6, v)
-				lh.AddRemote(n.Details.VpnIp, ans, false)
 		}
 	}
 
-	case NebulaMeta_HostMovedNotification:
+	// We prefer the first n addresses if we got too big
-	case NebulaMeta_HostPunchNotification:
+	if len(am.v4) > MaxRemotes {
-		if !lh.IsLighthouseIP(vpnIp) {
+		am.v4 = am.v4[:MaxRemotes]
+	}
+
+	if len(am.v6) > MaxRemotes {
+		am.v6 = am.v6[:MaxRemotes]
+	}
+}
+
+func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, vpnIp uint32, w EncWriter) {
+	if !lhh.lh.IsLighthouseIP(vpnIp) {
 		return
 	}
 
 	empty := []byte{0}
-		for _, a := range n.Details.IpAndPorts {
+	punch := func(vpnPeer *udpAddr) {
-			vpnPeer := NewUDPAddrFromLH4(a)
 		if vpnPeer == nil {
-				continue
+			return
 		}
 
 		go func() {
-				time.Sleep(lh.punchDelay)
+			time.Sleep(lhh.lh.punchDelay)
-				lh.metricHolepunchTx.Inc(1)
+			lhh.lh.metricHolepunchTx.Inc(1)
-				lh.punchConn.WriteTo(empty, vpnPeer)
+			lhh.lh.punchConn.WriteTo(empty, vpnPeer)
-
 		}()
 
-			if lh.l.Level >= logrus.DebugLevel {
+		if lhh.l.Level >= logrus.DebugLevel {
 			//TODO: lacking the ip we are actually punching on, old: l.Debugf("Punching %s on %d for %s", IntIp(a.Ip), a.Port, IntIp(n.Details.VpnIp))
-				lh.l.Debugf("Punching on %d for %s", a.Port, IntIp(n.Details.VpnIp))
+			lhh.l.Debugf("Punching on %d for %s", vpnPeer.Port, IntIp(n.Details.VpnIp))
 		}
 	}
 
+	for _, a := range n.Details.Ip4AndPorts {
+		punch(NewUDPAddrFromLH4(a))
+	}
+
 	for _, a := range n.Details.Ip6AndPorts {
-			vpnPeer := NewUDPAddrFromLH6(a)
+		punch(NewUDPAddrFromLH6(a))
-			if vpnPeer == nil {
-				continue
-			}
-
-			go func() {
-				time.Sleep(lh.punchDelay)
-				lh.metricHolepunchTx.Inc(1)
-				lh.punchConn.WriteTo(empty, vpnPeer)
-
-			}()
-
-			if lh.l.Level >= logrus.DebugLevel {
-				//TODO: lacking the ip we are actually punching on, old: l.Debugf("Punching %s on %d for %s", IntIp(a.Ip), a.Port, IntIp(n.Details.VpnIp))
-				lh.l.Debugf("Punching on %d for %s", a.Port, IntIp(n.Details.VpnIp))
-			}
 	}
 
 	// This sends a nebula test packet to the host trying to contact us. In the case
 	// of a double nat or other difficult scenario, this may help establish
 	// a tunnel.
-		if lh.punchBack {
+	if lhh.lh.punchBack {
 		go func() {
 			time.Sleep(time.Second * 5)
-				lh.l.Debugf("Sending a nebula test packet to vpn ip %s", IntIp(n.Details.VpnIp))
+			if lhh.l.Level >= logrus.DebugLevel {
-				// TODO we have to allocate a new output buffer here since we are spawning a new goroutine
+				lhh.l.Debugf("Sending a nebula test packet to vpn ip %s", IntIp(n.Details.VpnIp))
+			}
+			//NOTE: we have to allocate a new output buffer here since we are spawning a new goroutine
 			// for each punchBack packet. We should move this into a timerwheel or a single goroutine
 			// managed by a channel.
-				f.SendMessageToVpnIp(test, testRequest, n.Details.VpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+			w.SendMessageToVpnIp(test, testRequest, n.Details.VpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
 		}()
 	}
 }
+
+func TransformLHReplyToUdpAddrs(ips *ip4And6) []*udpAddr {
+	addrs := make([]*udpAddr, len(ips.v4)+len(ips.v6)+len(ips.learnedV4)+len(ips.learnedV6))
+	i := 0
+
+	for _, v := range ips.learnedV4 {
+		addrs[i] = NewUDPAddrFromLH4(v)
+		i++
 	}
 
-func (lh *LightHouse) metricRx(t NebulaMeta_MessageType, i int64) {
+	for _, v := range ips.v4 {
-	lh.metrics.Rx(NebulaMessageType(t), 0, i)
+		addrs[i] = NewUDPAddrFromLH4(v)
-}
+		i++
-func (lh *LightHouse) metricTx(t NebulaMeta_MessageType, i int64) {
-	lh.metrics.Tx(NebulaMessageType(t), 0, i)
 	}
 
-/*
+	for _, v := range ips.learnedV6 {
-func (f *Interface) sendPathCheck(ci *ConnectionState, endpoint *net.UDPAddr, counter int) {
+		addrs[i] = NewUDPAddrFromLH6(v)
-	c := ci.messageCounter
+		i++
-    b := HeaderEncode(nil, Version, uint8(path_check), 0, ci.remoteIndex, c)
-	ci.messageCounter++
-
-	if ci.eKey != nil {
-		msg := ci.eKey.EncryptDanger(b, nil, []byte(strconv.Itoa(counter)), c)
-		//msg := ci.eKey.EncryptDanger(b, nil, []byte(fmt.Sprintf("%d", counter)), c)
-		f.outside.WriteTo(msg, endpoint)
-		l.Debugf("path_check sent, remote index: %d, pathCounter %d", ci.remoteIndex, counter)
-	}
 	}
 
-func (f *Interface) sendPathCheckReply(ci *ConnectionState, endpoint *net.UDPAddr, counter []byte) {
+	for _, v := range ips.v6 {
-	c := ci.messageCounter
+		addrs[i] = NewUDPAddrFromLH6(v)
-    b := HeaderEncode(nil, Version, uint8(path_check_reply), 0, ci.remoteIndex, c)
+		i++
-	ci.messageCounter++
+	}
 
-	if ci.eKey != nil {
+	return addrs
-		msg := ci.eKey.EncryptDanger(b, nil, counter, c)
-		f.outside.WriteTo(msg, endpoint)
-		l.Debugln("path_check sent, remote index: ", ci.remoteIndex)
 }
-}
-*/

lighthouse_test.go

@@ -1,6 +1,7 @@
 package nebula
 
 import (
+	"fmt"
 	"net"
 	"testing"
 
@@ -13,7 +14,7 @@ import (
 func TestOldIPv4Only(t *testing.T) {
 	// This test ensures our new ipv6 enabled LH protobuf IpAndPorts works with the old style to enable backwards compatibility
 	b := []byte{8, 129, 130, 132, 80, 16, 10}
-	var m IpAndPort
+	var m Ip4AndPort
 	err := proto.Unmarshal(b, &m)
 	assert.NoError(t, err)
 	assert.Equal(t, "10.1.1.1", int2ip(m.GetIp()).String())
@@ -40,30 +41,6 @@ func TestNewLhQuery(t *testing.T) {
 
 }
 
-func TestNewipandportfromudpaddr(t *testing.T) {
-	blah := NewUDPAddrFromString("1.2.2.3:12345")
-	meh := NewIpAndPortFromUDPAddr(blah)
-	assert.Equal(t, uint32(16908803), meh.v4.Ip)
-	assert.Equal(t, uint32(12345), meh.v4.Port)
-}
-
-func TestSetipandportsfromudpaddrs(t *testing.T) {
-	blah := NewUDPAddrFromString("1.2.2.3:12345")
-	blah2 := NewUDPAddrFromString("9.9.9.9:47828")
-	group := []*udpAddr{blah, blah2}
-	var lh *LightHouse
-	lhh := lh.NewRequestHandler()
-	result := lhh.setIpAndPortsFromNetIps(group)
-	assert.IsType(t, []*ip4Or6{}, result)
-	assert.Len(t, result, 2)
-	assert.Equal(t, uint32(0x01020203), result[0].v4.Ip)
-	assert.Equal(t, uint32(12345), result[0].v4.Port)
-	assert.Equal(t, uint32(0x09090909), result[1].v4.Ip)
-	assert.Equal(t, uint32(47828), result[1].v4.Port)
-	//t.Error(reflect.TypeOf(hah))
-
-}
-
 func Test_lhStaticMapping(t *testing.T) {
 	l := NewTestLogger()
 	lh1 := "10.128.0.2"
@@ -96,11 +73,17 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) {
 
 	hAddr := NewUDPAddrFromString("4.5.6.7:12345")
 	hAddr2 := NewUDPAddrFromString("4.5.6.7:12346")
-	lh.addrMap[3] = []*udpAddr{hAddr, hAddr2}
+	lh.addrMap[3] = &ip4And6{v4: []*Ip4AndPort{
+		NewIp4AndPort(hAddr.IP, uint32(hAddr.Port)),
+		NewIp4AndPort(hAddr2.IP, uint32(hAddr2.Port))},
+	}
 
 	rAddr := NewUDPAddrFromString("1.2.2.3:12345")
 	rAddr2 := NewUDPAddrFromString("1.2.2.3:12346")
-	lh.addrMap[2] = []*udpAddr{rAddr, rAddr2}
+	lh.addrMap[2] = &ip4And6{v4: []*Ip4AndPort{
+		NewIp4AndPort(rAddr.IP, uint32(rAddr.Port)),
+		NewIp4AndPort(rAddr2.IP, uint32(rAddr2.Port))},
+	}
 
 	mw := &mockEncWriter{}
 
@@ -110,13 +93,13 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) {
 			Type: NebulaMeta_HostQuery,
 			Details: &NebulaMetaDetails{
 				VpnIp:       4,
-				IpAndPorts: nil,
+				Ip4AndPorts: nil,
 			},
 		}
 		p, err := proto.Marshal(req)
 		assert.NoError(b, err)
 		for n := 0; n < b.N; n++ {
-			lhh.HandleRequest(rAddr, 2, p, nil, mw)
+			lhh.HandleRequest(rAddr, 2, p, mw)
 		}
 	})
 	b.Run("found", func(b *testing.B) {
@@ -125,18 +108,138 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) {
 			Type: NebulaMeta_HostQuery,
 			Details: &NebulaMetaDetails{
 				VpnIp:       3,
-				IpAndPorts: nil,
+				Ip4AndPorts: nil,
 			},
 		}
 		p, err := proto.Marshal(req)
 		assert.NoError(b, err)
 
 		for n := 0; n < b.N; n++ {
-			lhh.HandleRequest(rAddr, 2, p, nil, mw)
+			lhh.HandleRequest(rAddr, 2, p, mw)
 		}
 	})
 }
 
+func TestLighthouse_Memory(t *testing.T) {
+	l := NewTestLogger()
+
+	myUdpAddr0 := &udpAddr{IP: net.ParseIP("10.0.0.2"), Port: 4242}
+	myUdpAddr1 := &udpAddr{IP: net.ParseIP("192.168.0.2"), Port: 4242}
+	myUdpAddr2 := &udpAddr{IP: net.ParseIP("172.16.0.2"), Port: 4242}
+	myUdpAddr3 := &udpAddr{IP: net.ParseIP("100.152.0.2"), Port: 4242}
+	myUdpAddr4 := &udpAddr{IP: net.ParseIP("24.15.0.2"), Port: 4242}
+	myUdpAddr5 := &udpAddr{IP: net.ParseIP("192.168.0.2"), Port: 4243}
+	myUdpAddr6 := &udpAddr{IP: net.ParseIP("192.168.0.2"), Port: 4244}
+	myUdpAddr7 := &udpAddr{IP: net.ParseIP("192.168.0.2"), Port: 4245}
+	myUdpAddr8 := &udpAddr{IP: net.ParseIP("192.168.0.2"), Port: 4246}
+	myUdpAddr9 := &udpAddr{IP: net.ParseIP("192.168.0.2"), Port: 4247}
+	myUdpAddr10 := &udpAddr{IP: net.ParseIP("192.168.0.2"), Port: 4248}
+	myUdpAddr11 := &udpAddr{IP: net.ParseIP("192.168.0.2"), Port: 4249}
+	myVpnIp := ip2int(net.ParseIP("10.128.0.2"))
+
+	theirUdpAddr0 := &udpAddr{IP: net.ParseIP("10.0.0.3"), Port: 4242}
+	theirUdpAddr1 := &udpAddr{IP: net.ParseIP("192.168.0.3"), Port: 4242}
+	theirUdpAddr2 := &udpAddr{IP: net.ParseIP("172.16.0.3"), Port: 4242}
+	theirUdpAddr3 := &udpAddr{IP: net.ParseIP("100.152.0.3"), Port: 4242}
+	theirUdpAddr4 := &udpAddr{IP: net.ParseIP("24.15.0.3"), Port: 4242}
+	theirVpnIp := ip2int(net.ParseIP("10.128.0.3"))
+
+	lhIP := net.ParseIP("10.128.0.1")
+	udpServer, _ := NewListener(l, "0.0.0.0", 0, true)
+	lh := NewLightHouse(l, true, 1, []uint32{ip2int(lhIP)}, 10, 10003, udpServer, false, 1, false)
+	lhh := lh.NewRequestHandler()
+
+	// Test that my first update responds with just that
+	newLHHostUpdate(myUdpAddr0, myVpnIp, []*udpAddr{myUdpAddr1, myUdpAddr2}, lhh)
+	r := newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh)
+	assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr1, myUdpAddr2)
+
+	// Ensure we don't accumulate addresses
+	newLHHostUpdate(myUdpAddr0, myVpnIp, []*udpAddr{myUdpAddr3}, lhh)
+	r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh)
+	assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr3)
+
+	// Grow it back to 2
+	newLHHostUpdate(myUdpAddr0, myVpnIp, []*udpAddr{myUdpAddr1, myUdpAddr4}, lhh)
+	r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh)
+	assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr1, myUdpAddr4)
+
+	// Update a different host
+	newLHHostUpdate(theirUdpAddr0, theirVpnIp, []*udpAddr{theirUdpAddr1, theirUdpAddr2, theirUdpAddr3, theirUdpAddr4}, lhh)
+	r = newLHHostRequest(theirUdpAddr0, theirVpnIp, myVpnIp, lhh)
+	assertIp4InArray(t, r.msg.Details.Ip4AndPorts, theirUdpAddr1, theirUdpAddr2, theirUdpAddr3, theirUdpAddr4)
+
+	// Make sure we didn't get changed
+	r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh)
+	assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr1, myUdpAddr4)
+
+	// Finally ensure proper ordering and limiting
+	// Send 12 addrs, get 10 back, one removed on a dupe check the other by limiting
+	newLHHostUpdate(
+		myUdpAddr0,
+		myVpnIp,
+		[]*udpAddr{
+			myUdpAddr1,
+			myUdpAddr2,
+			myUdpAddr3,
+			myUdpAddr4,
+			myUdpAddr5,
+			myUdpAddr5, //Duplicated on purpose
+			myUdpAddr6,
+			myUdpAddr7,
+			myUdpAddr8,
+			myUdpAddr9,
+			myUdpAddr10,
+			myUdpAddr11, // This should get cut
+		}, lhh)
+	r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh)
+	assertIp4InArray(
+		t,
+		r.msg.Details.Ip4AndPorts,
+		myUdpAddr1, myUdpAddr2, myUdpAddr3, myUdpAddr4, myUdpAddr5, myUdpAddr6, myUdpAddr7, myUdpAddr8, myUdpAddr9, myUdpAddr10,
+	)
+}
+
+func newLHHostRequest(fromAddr *udpAddr, myVpnIp, queryVpnIp uint32, lhh *LightHouseHandler) testLhReply {
+	req := &NebulaMeta{
+		Type: NebulaMeta_HostQuery,
+		Details: &NebulaMetaDetails{
+			VpnIp: queryVpnIp,
+		},
+	}
+
+	b, err := req.Marshal()
+	if err != nil {
+		panic(err)
+	}
+
+	w := &testEncWriter{}
+	lhh.HandleRequest(fromAddr, myVpnIp, b, w)
+	return w.lastReply
+}
+
+func newLHHostUpdate(fromAddr *udpAddr, vpnIp uint32, addrs []*udpAddr, lhh *LightHouseHandler) {
+	req := &NebulaMeta{
+		Type: NebulaMeta_HostUpdateNotification,
+		Details: &NebulaMetaDetails{
+			VpnIp:       vpnIp,
+			Ip4AndPorts: make([]*Ip4AndPort, len(addrs)),
+		},
+	}
+
+	for k, v := range addrs {
+		req.Details.Ip4AndPorts[k] = &Ip4AndPort{Ip: ip2int(v.IP), Port: uint32(v.Port)}
+	}
+
+	b, err := req.Marshal()
+	if err != nil {
+		panic(err)
+	}
+
+	w := &testEncWriter{}
+	lhh.HandleRequest(fromAddr, vpnIp, b, w)
+}
+
 func Test_lhRemoteAllowList(t *testing.T) {
 	l := NewTestLogger()
 	c := NewConfig(l)
@@ -154,48 +257,94 @@ func Test_lhRemoteAllowList(t *testing.T) {
 	lh := NewLightHouse(l, true, 1, []uint32{ip2int(lh1IP)}, 10, 10003, udpServer, false, 1, false)
 	lh.SetRemoteAllowList(allowList)
 
-	remote1 := "10.20.0.3"
+	// A disallowed ip should not enter the cache but we should end up with an empty entry in the addrMap
-	remote1IP := net.ParseIP(remote1)
+	remote1IP := net.ParseIP("10.20.0.3")
 	lh.AddRemote(ip2int(remote1IP), NewUDPAddr(remote1IP, uint16(4242)), true)
-	assert.Nil(t, lh.addrMap[ip2int(remote1IP)])
+	assert.NotNil(t, lh.addrMap[ip2int(remote1IP)])
+	assert.Empty(t, lh.addrMap[ip2int(remote1IP)].v4)
+	assert.Empty(t, lh.addrMap[ip2int(remote1IP)].v6)
 
-	remote2 := "10.128.0.3"
+	// Make sure a good ip enters the cache and addrMap
-	remote2IP := net.ParseIP(remote2)
+	remote2IP := net.ParseIP("10.128.0.3")
 	remote2UDPAddr := NewUDPAddr(remote2IP, uint16(4242))
-
 	lh.AddRemote(ip2int(remote2IP), remote2UDPAddr, true)
-	// Make sure the pointers are different but the contents are equal since we are using slices
+	assertIp4InArray(t, lh.addrMap[ip2int(remote2IP)].learnedV4, remote2UDPAddr)
-	assert.False(t, remote2UDPAddr == lh.addrMap[ip2int(remote2IP)][0])
+
-	assert.Equal(t, remote2UDPAddr, lh.addrMap[ip2int(remote2IP)][0])
+	// Another good ip gets into the cache, ordering is inverted
+	remote3IP := net.ParseIP("10.128.0.4")
+	remote3UDPAddr := NewUDPAddr(remote3IP, uint16(4243))
+	lh.AddRemote(ip2int(remote2IP), remote3UDPAddr, true)
+	assertIp4InArray(t, lh.addrMap[ip2int(remote2IP)].learnedV4, remote3UDPAddr, remote2UDPAddr)
+
+	// If we exceed the length limit we should only have the most recent addresses
+	addedAddrs := []*udpAddr{}
+	for i := 0; i < 11; i++ {
+		remoteUDPAddr := NewUDPAddr(net.IP{10, 128, 0, 4}, uint16(4243+i))
+		lh.AddRemote(ip2int(remote2IP), remoteUDPAddr, true)
+		// The first entry here is a duplicate, don't add it to the assert list
+		if i != 0 {
+			addedAddrs = append(addedAddrs, remoteUDPAddr)
+		}
 	}
 
-//func NewLightHouse(amLighthouse bool, myIp uint32, ips []string, interval int, nebulaPort int, pc *udpConn, punchBack bool) *LightHouse {
+	// We should only have the last 10 of what we tried to add
+	assert.True(t, len(addedAddrs) >= 10, "We should have tried to add at least 10 addresses")
+	ln := len(addedAddrs)
+	assertIp4InArray(
+		t,
+		lh.addrMap[ip2int(remote2IP)].learnedV4,
+		addedAddrs[ln-1],
+		addedAddrs[ln-2],
+		addedAddrs[ln-3],
+		addedAddrs[ln-4],
+		addedAddrs[ln-5],
+		addedAddrs[ln-6],
+		addedAddrs[ln-7],
+		addedAddrs[ln-8],
+		addedAddrs[ln-9],
+		addedAddrs[ln-10],
+	)
+}
 
-/*
+type testLhReply struct {
-func TestLHQuery(t *testing.T) {
+	nebType    NebulaMessageType
-	//n := NewLhQueryByIpString("10.128.0.3")
+	nebSubType NebulaMessageSubType
-	_, myNet, _ := net.ParseCIDR("10.128.0.0/16")
+	vpnIp      uint32
-	m := NewHostMap(myNet)
+	msg        *NebulaMeta
-	y, _ := net.ResolveUDPAddr("udp", "10.128.0.3:11111")
+}
-	m.Add(ip2int(net.ParseIP("127.0.0.1")), y)
-	//t.Errorf("%s", m)
-	_ = m
 
-	_, n, _ := net.ParseCIDR("127.0.0.1/8")
+type testEncWriter struct {
+	lastReply testLhReply
+}
 
-	/*udpServer, err := net.ListenUDP("udp", &net.UDPAddr{Port: 10009})
+func (tw *testEncWriter) SendMessageToVpnIp(t NebulaMessageType, st NebulaMessageSubType, vpnIp uint32, p, _, _ []byte) {
+	tw.lastReply = testLhReply{
+		nebType:    t,
+		nebSubType: st,
+		vpnIp:      vpnIp,
+		msg:        &NebulaMeta{},
+	}
+
+	err := proto.Unmarshal(p, tw.lastReply.msg)
 	if err != nil {
-		t.Errorf("%s", err)
+		panic(err)
+	}
 }
 
-	meh := NewLightHouse(n, m, []string{"10.128.0.2"}, false, 10, 10003, 10004)
+// assertIp4InArray asserts every address in want is at the same position in have and that the lengths match
-	//t.Error(m.Hosts)
+func assertIp4InArray(t *testing.T, have []*Ip4AndPort, want ...*udpAddr) {
-	meh2, err := meh.Query(ip2int(net.ParseIP("10.128.0.3")))
+	assert.Len(t, have, len(want))
-	t.Error(err)
+	for k, w := range want {
-	if err != nil {
+		if !(have[k].Ip == ip2int(w.IP) && have[k].Port == uint32(w.Port)) {
-		return
+			assert.Fail(t, fmt.Sprintf("Response did not contain: %v:%v at %v; %v", w.IP, w.Port, k, translateV4toUdpAddr(have)))
 		}
-	t.Errorf("%s", meh2)
-	t.Errorf("%s", n)
 	}
-*/
+}
+
+func translateV4toUdpAddr(ips []*Ip4AndPort) []*udpAddr {
+	addrs := make([]*udpAddr, len(ips))
+	for k, v := range ips {
+		addrs[k] = NewUDPAddrFromLH4(v)
+	}
+	return addrs
+}

nebula.proto

@@ -1,8 +1,6 @@
 syntax = "proto3";
 package nebula;
 
-option go_package = "github.com/slackhq/nebula";
-
 message NebulaMeta {
   enum MessageType {
     None = 0;
@@ -24,12 +22,12 @@ message NebulaMeta {
 
 message NebulaMetaDetails {
   uint32 VpnIp = 1;
-  repeated IpAndPort IpAndPorts = 2;
+  repeated Ip4AndPort Ip4AndPorts = 2;
   repeated Ip6AndPort Ip6AndPorts = 4;
   uint32 counter = 3;
 }
 
-message IpAndPort {
+message Ip4AndPort {
   uint32 Ip = 1;
   uint32 Port = 2;
 }

outside.go

@@ -66,7 +66,7 @@ func (f *Interface) readOutsidePackets(addr *udpAddr, out []byte, packet []byte,
 			return
 		}
 
-		lhh.HandleRequest(addr, hostinfo.hostId, d, hostinfo.GetCert(), f)
+		lhh.HandleRequest(addr, hostinfo.hostId, d, f)
 
 		// Fallthrough to the bottom to record incoming traffic
 

ssh.go

@@ -402,14 +402,9 @@ func sshListLighthouseMap(lightHouse *LightHouse, a interface{}, w sshd.StringWr
 		x := 0
 		var h m
 		for vpnIp, v := range lightHouse.addrMap {
-			ips := make([]string, len(v))
-			for i, ip := range v {
-				ips[i] = ip.String()
-			}
-
 			h = m{
 				"vpnIp": int2ip(vpnIp),
-				"addrs": ips,
+				"addrs": TransformLHReplyToUdpAddrs(v),
 			}
 
 			d[x] = h
@@ -423,11 +418,7 @@ func sshListLighthouseMap(lightHouse *LightHouse, a interface{}, w sshd.StringWr
 		}
 	} else {
 		for vpnIp, v := range lightHouse.addrMap {
-			ips := make([]string, len(v))
+			err := w.WriteLine(fmt.Sprintf("%s: %s", int2ip(vpnIp), TransformLHReplyToUdpAddrs(v)))
-			for i, ip := range v {
-				ips[i] = ip.String()
-			}
-			err := w.WriteLine(fmt.Sprintf("%s: %s", int2ip(vpnIp), ips))
 			if err != nil {
 				return err
 			}