diff --git a/.github/workflows/smoke/smoke.sh b/.github/workflows/smoke/smoke.sh
index 591922c..df73ea1 100755
--- a/.github/workflows/smoke/smoke.sh
+++ b/.github/workflows/smoke/smoke.sh
@@ -2,6 +2,10 @@
 
 set -e -x
 
+docker run --name lighthouse1 --rm nebula:smoke -config lighthouse1.yml -test
+docker run --name host2 --rm nebula:smoke -config host2.yml -test
+docker run --name host3 --rm nebula:smoke -config host3.yml -test
+
 docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config lighthouse1.yml &
 sleep 1
 docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host2.yml &
diff --git a/main.go b/main.go
index 264274a..e34c51b 100644
--- a/main.go
+++ b/main.go
@@ -101,32 +101,35 @@ func Main(configPath string, configTest bool, buildVersion string) {
 	// tun config, listeners, anything modifying the computer should be below
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-	if configTest {
-		os.Exit(0)
-	}
+	var tun *Tun
+	if !configTest {
+		config.CatchHUP()
 
-	config.CatchHUP()
-
-	// set up our tun dev
-	tun, err := newTun(
-		config.GetString("tun.dev", ""),
-		tunCidr,
-		config.GetInt("tun.mtu", DEFAULT_MTU),
-		routes,
-		unsafeRoutes,
-		config.GetInt("tun.tx_queue", 500),
-	)
-	if err != nil {
-		l.WithError(err).Fatal("Failed to get a tun/tap device")
+		// set up our tun dev
+		tun, err = newTun(
+			config.GetString("tun.dev", ""),
+			tunCidr,
+			config.GetInt("tun.mtu", DEFAULT_MTU),
+			routes,
+			unsafeRoutes,
+			config.GetInt("tun.tx_queue", 500),
+		)
+		if err != nil {
+			l.WithError(err).Fatal("Failed to get a tun/tap device")
+		}
 	}
 
 	// set up our UDP listener
 	udpQueues := config.GetInt("listen.routines", 1)
-	udpServer, err := NewListener(config.GetString("listen.host", "0.0.0.0"), config.GetInt("listen.port", 0), udpQueues > 1)
-	if err != nil {
-		l.WithError(err).Fatal("Failed to open udp listener")
+	var udpServer *udpConn
+
+	if !configTest {
+		udpServer, err = NewListener(config.GetString("listen.host", "0.0.0.0"), config.GetInt("listen.port", 0), udpQueues > 1)
+		if err != nil {
+			l.WithError(err).Fatal("Failed to open udp listener")
+		}
+		udpServer.reloadConfig(config)
 	}
-	udpServer.reloadConfig(config)
 
 	// Set up my internal host map
 	var preferredRanges []*net.IPNet
@@ -178,14 +181,14 @@ func Main(configPath string, configTest bool, buildVersion string) {
 	*/
 
 	punchy := NewPunchyFromConfig(config)
-	if punchy.Punch {
+	if punchy.Punch && !configTest {
 		l.Info("UDP hole punching enabled")
 		go hostMap.Punchy(udpServer)
 	}
 
 	port := config.GetInt("listen.port", 0)
 	// If port is dynamic, discover it
-	if port == 0 {
+	if port == 0 && !configTest {
 		uPort, err := udpServer.LocalAddr()
 		if err != nil {
 			l.WithError(err).Fatal("Failed to get listening port")
@@ -306,21 +309,28 @@ func Main(configPath string, configTest bool, buildVersion string) {
 		l.Fatalf("Unknown cipher: %v", ifConfig.Cipher)
 	}
 
-	ifce, err := NewInterface(ifConfig)
-	if err != nil {
-		l.WithError(err).Fatal("Failed to initialize interface")
+	var ifce *Interface
+	if !configTest {
+		ifce, err = NewInterface(ifConfig)
+		if err != nil {
+			l.WithError(err).Fatal("Failed to initialize interface")
+		}
+
+		ifce.RegisterConfigChangeCallbacks(config)
+
+		go handshakeManager.Run(ifce)
+		go lightHouse.LhUpdateWorker(ifce)
 	}
 
-	ifce.RegisterConfigChangeCallbacks(config)
-
-	go handshakeManager.Run(ifce)
-	go lightHouse.LhUpdateWorker(ifce)
-
-	err = startStats(config)
+	err = startStats(config, configTest)
 	if err != nil {
 		l.WithError(err).Fatal("Failed to start stats emitter")
 	}
 
+	if configTest {
+		os.Exit(0)
+	}
+
 	//TODO: check if we _should_ be emitting stats
 	go ifce.emitStats(config.GetDuration("stats.interval", time.Second*10))
 
diff --git a/stats.go b/stats.go
index 65ad877..dfc1931 100644
--- a/stats.go
+++ b/stats.go
@@ -14,7 +14,7 @@ import (
 	"time"
 )
 
-func startStats(c *Config) error {
+func startStats(c *Config, configTest bool) error {
 	mType := c.GetString("stats.type", "")
 	if mType == "" || mType == "none" {
 		return nil
@@ -27,9 +27,9 @@ func startStats(c *Config) error {
 
 	switch mType {
 	case "graphite":
-		startGraphiteStats(interval, c)
+		startGraphiteStats(interval, c, configTest)
 	case "prometheus":
-		startPrometheusStats(interval, c)
+		startPrometheusStats(interval, c, configTest)
 	default:
 		return fmt.Errorf("stats.type was not understood: %s", mType)
 	}
@@ -43,7 +43,7 @@ func startStats(c *Config) error {
 	return nil
 }
 
-func startGraphiteStats(i time.Duration, c *Config) error {
+func startGraphiteStats(i time.Duration, c *Config, configTest bool) error {
 	proto := c.GetString("stats.protocol", "tcp")
 	host := c.GetString("stats.host", "")
 	if host == "" {
@@ -57,11 +57,13 @@ func startGraphiteStats(i time.Duration, c *Config) error {
 	}
 
 	l.Infof("Starting graphite. Interval: %s, prefix: %s, addr: %s", i, prefix, addr)
-	go graphite.Graphite(metrics.DefaultRegistry, i, prefix, addr)
+	if !configTest {
+		go graphite.Graphite(metrics.DefaultRegistry, i, prefix, addr)
+	}
 	return nil
 }
 
-func startPrometheusStats(i time.Duration, c *Config) error {
+func startPrometheusStats(i time.Duration, c *Config, configTest bool) error {
 	namespace := c.GetString("stats.namespace", "")
 	subsystem := c.GetString("stats.subsystem", "")
 
@@ -79,11 +81,13 @@ func startPrometheusStats(i time.Duration, c *Config) error {
 	pClient := mp.NewPrometheusProvider(metrics.DefaultRegistry, namespace, subsystem, pr, i)
 	go pClient.UpdatePrometheusMetrics()
 
-	go func() {
-		l.Infof("Prometheus stats listening on %s at %s", listen, path)
-		http.Handle(path, promhttp.HandlerFor(pr, promhttp.HandlerOpts{ErrorLog: l}))
-		log.Fatal(http.ListenAndServe(listen, nil))
-	}()
+	if !configTest {
+		go func() {
+			l.Infof("Prometheus stats listening on %s at %s", listen, path)
+			http.Handle(path, promhttp.HandlerFor(pr, promhttp.HandlerOpts{ErrorLog: l}))
+			log.Fatal(http.ListenAndServe(listen, nil))
+		}()
+	}
 
 	return nil
 }