diff --git a/go.mod b/go.mod index 420c675..c789cc9 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,13 @@ module github.com/costela/wesher require ( + github.com/cenkalti/backoff v2.2.1+incompatible github.com/hashicorp/errwrap v1.0.0 github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/go-sockaddr v1.0.0 github.com/hashicorp/memberlist v0.1.3 github.com/mattn/go-isatty v0.0.7 - github.com/mdlayher/genetlink v0.0.0-20190617154021-985b2115c31a + github.com/mdlayher/genetlink v0.0.0-20190617154021-985b2115c31a // indirect github.com/pkg/errors v0.8.1 github.com/sirupsen/logrus v1.3.0 github.com/stevenroose/gonfig v0.1.4 diff --git a/go.sum b/go.sum index 489dc74..ea40b7f 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -41,7 +43,6 @@ github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE github.com/mdlayher/netlink v0.0.0-20190513144208-ba284d510044/go.mod h1:gOrA34zDL0K3RsACQe54bDYLF/CeFspQ9m5DOycycQ8= github.com/mdlayher/netlink v0.0.0-20190614145538-d8264f87dbe3 h1:3IPcWjiboJFnnvHeXxT4pYw33BiPJn/DC5BKhcGEbGk= github.com/mdlayher/netlink v0.0.0-20190614145538-d8264f87dbe3/go.mod h1:ISujvOTprADlNr00kvJIu0d23q57wk2NSV/PT/TEk4E= -github.com/mdlayher/netlink v0.0.0-20190617153422-f82a9b10b2bc h1:deLjDmcgzsCAO+7m2aeuyhQCqvn1LuCCSMLWUARnad8= github.com/microsoft/go-winio v0.4.12 h1:3vDRRsUnj2dKE7QKoedntu9hbuD8gzaVd2E2UZioqx4= github.com/microsoft/go-winio v0.4.12/go.mod h1:kcIxxtKZE55DEncT/EOvFiygPobhUWpSDqDb47poQOU= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= diff --git a/main.go b/main.go index 31c6168..492fba3 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,9 @@ import ( "os" "os/signal" "syscall" + "time" + + "github.com/cenkalti/backoff" "github.com/sirupsen/logrus" @@ -39,7 +42,13 @@ func main() { } nodec, errc := cluster.members() // avoid deadlocks by starting before join - if err := cluster.join(config.Join); err != nil { + if err := backoff.RetryNotify( + func() error { return cluster.join(config.Join) }, + backoff.NewExponentialBackOff(), + func(err error, dur time.Duration) { + logrus.Errorf("could not join cluster, retrying in %s: %s", dur, err) + }, + ); err != nil { logrus.Fatalf("could not join cluster: %s", err) } diff --git a/tests/e2e.sh b/tests/e2e.sh index 0f3d1ce..d8be1c1 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -64,6 +64,21 @@ test_node_restart() { stop_test_container test1-orig } +test_cluster_simultaneous_start() { + run_test_container test1-orig test1 --join test2-orig,test3-orig + run_test_container test2-orig test2 --join test1-orig,test3-orig + run_test_container test3-orig test3 --join test1-orig,test2-orig + + sleep 3 + + docker exec test1-orig ping -c1 -W1 test2 || (docker logs test1-orig; docker logs test2-orig; false) + docker exec test1-orig ping -c1 -W1 test3 || (docker logs test1-orig; docker logs test3-orig; false) + + stop_test_container test3-orig + stop_test_container test2-orig + stop_test_container test1-orig +} + for test_func in $(declare -F | grep -Eo '\