From 3a03d3683ebea5d6527f3f86e6a77ec2e03a2de2 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Tue, 17 Oct 2017 16:00:31 -0400 Subject: [PATCH 1/2] update consul api packages This is from a commit just after the v1.0.0 release, because it removes the Porter service dependency for tests. The client api package was not changed. --- backend/remote-state/consul/backend_test.go | 6 +- vendor/github.com/hashicorp/consul/api/acl.go | 18 +++ .../github.com/hashicorp/consul/api/agent.go | 152 +++++++++++++++++- vendor/github.com/hashicorp/consul/api/api.go | 23 ++- .../hashicorp/consul/api/coordinate.go | 5 +- vendor/github.com/hashicorp/consul/api/kv.go | 30 ++-- .../github.com/hashicorp/consul/api/lock.go | 4 +- .../hashicorp/consul/api/operator_area.go | 25 +++ .../consul/api/operator_autopilot.go | 2 +- .../hashicorp/consul/api/operator_keyring.go | 3 + .../hashicorp/consul/api/operator_raft.go | 5 +- .../hashicorp/consul/api/operator_segment.go | 11 ++ .../hashicorp/consul/api/semaphore.go | 4 +- .../hashicorp/consul/lib/freeport/freeport.go | 139 ++++++++++++++++ .../hashicorp/consul/testutil/server.go | 49 +++--- .../consul/testutil/server_methods.go | 4 +- vendor/vendor.json | 34 ++-- 17 files changed, 434 insertions(+), 80 deletions(-) create mode 100644 vendor/github.com/hashicorp/consul/api/operator_segment.go create mode 100644 vendor/github.com/hashicorp/consul/lib/freeport/freeport.go diff --git a/backend/remote-state/consul/backend_test.go b/backend/remote-state/consul/backend_test.go index 7c4bf5ee5..6beda5f5b 100644 --- a/backend/remote-state/consul/backend_test.go +++ b/backend/remote-state/consul/backend_test.go @@ -22,7 +22,7 @@ func newConsulTestServer(t *testing.T) *testutil.TestServer { t.Skip() } - srv, _ := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + srv, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { c.LogLevel = "warn" if !testing.Verbose() { @@ -31,6 +31,10 @@ func newConsulTestServer(t *testing.T) *testutil.TestServer { } }) + if err != nil { + t.Fatal(err) + } + return srv } diff --git a/vendor/github.com/hashicorp/consul/api/acl.go b/vendor/github.com/hashicorp/consul/api/acl.go index 15d1f9f5a..6ea0a752e 100644 --- a/vendor/github.com/hashicorp/consul/api/acl.go +++ b/vendor/github.com/hashicorp/consul/api/acl.go @@ -42,6 +42,24 @@ func (c *Client) ACL() *ACL { return &ACL{c} } +// Bootstrap is used to perform a one-time ACL bootstrap operation on a cluster +// to get the first management token. +func (a *ACL) Bootstrap() (string, *WriteMeta, error) { + r := a.c.newRequest("PUT", "/v1/acl/bootstrap") + rtt, resp, err := requireOK(a.c.doRequest(r)) + if err != nil { + return "", nil, err + } + defer resp.Body.Close() + + wm := &WriteMeta{RequestTime: rtt} + var out struct{ ID string } + if err := decodeBody(resp, &out); err != nil { + return "", nil, err + } + return out.ID, wm, nil +} + // Create is used to generate a new token with the given parameters func (a *ACL) Create(acl *ACLEntry, q *WriteOptions) (string, *WriteMeta, error) { r := a.c.newRequest("PUT", "/v1/acl/create") diff --git a/vendor/github.com/hashicorp/consul/api/agent.go b/vendor/github.com/hashicorp/consul/api/agent.go index 605592db9..533b24557 100644 --- a/vendor/github.com/hashicorp/consul/api/agent.go +++ b/vendor/github.com/hashicorp/consul/api/agent.go @@ -44,6 +44,19 @@ type AgentMember struct { DelegateCur uint8 } +// AllSegments is used to select for all segments in MembersOpts. +const AllSegments = "_all" + +// MembersOpts is used for querying member information. +type MembersOpts struct { + // WAN is whether to show members from the WAN. + WAN bool + + // Segment is the LAN segment to show members for. Setting this to the + // AllSegments value above will show members in all segments. + Segment string +} + // AgentServiceRegistration is used to register a new service type AgentServiceRegistration struct { ID string `json:",omitempty"` @@ -67,7 +80,8 @@ type AgentCheckRegistration struct { // AgentServiceCheck is used to define a node or service level check type AgentServiceCheck struct { - Script string `json:",omitempty"` + Args []string `json:"ScriptArgs,omitempty"` + Script string `json:",omitempty"` // Deprecated, use Args. DockerContainerID string `json:",omitempty"` Shell string `json:",omitempty"` // Only supported for Docker. Interval string `json:",omitempty"` @@ -91,6 +105,47 @@ type AgentServiceCheck struct { } type AgentServiceChecks []*AgentServiceCheck +// AgentToken is used when updating ACL tokens for an agent. +type AgentToken struct { + Token string +} + +// Metrics info is used to store different types of metric values from the agent. +type MetricsInfo struct { + Timestamp string + Gauges []GaugeValue + Points []PointValue + Counters []SampledValue + Samples []SampledValue +} + +// GaugeValue stores one value that is updated as time goes on, such as +// the amount of memory allocated. +type GaugeValue struct { + Name string + Value float32 + Labels map[string]string +} + +// PointValue holds a series of points for a metric. +type PointValue struct { + Name string + Points []float32 +} + +// SampledValue stores info about a metric that is incremented over time, +// such as the number of requests to an HTTP endpoint. +type SampledValue struct { + Name string + Count int + Sum float64 + Min float64 + Max float64 + Mean float64 + Stddev float64 + Labels map[string]string +} + // Agent can be used to query the Agent endpoints type Agent struct { c *Client @@ -121,6 +176,23 @@ func (a *Agent) Self() (map[string]map[string]interface{}, error) { return out, nil } +// Metrics is used to query the agent we are speaking to for +// its current internal metric data +func (a *Agent) Metrics() (*MetricsInfo, error) { + r := a.c.newRequest("GET", "/v1/agent/metrics") + _, resp, err := requireOK(a.c.doRequest(r)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var out *MetricsInfo + if err := decodeBody(resp, &out); err != nil { + return nil, err + } + return out, nil +} + // Reload triggers a configuration reload for the agent we are connected to. func (a *Agent) Reload() error { r := a.c.newRequest("PUT", "/v1/agent/reload") @@ -198,6 +270,28 @@ func (a *Agent) Members(wan bool) ([]*AgentMember, error) { return out, nil } +// MembersOpts returns the known gossip members and can be passed +// additional options for WAN/segment filtering. +func (a *Agent) MembersOpts(opts MembersOpts) ([]*AgentMember, error) { + r := a.c.newRequest("GET", "/v1/agent/members") + r.params.Set("segment", opts.Segment) + if opts.WAN { + r.params.Set("wan", "1") + } + + _, resp, err := requireOK(a.c.doRequest(r)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var out []*AgentMember + if err := decodeBody(resp, &out); err != nil { + return nil, err + } + return out, nil +} + // ServiceRegister is used to register a new service with // the local agent func (a *Agent) ServiceRegister(service *AgentServiceRegistration) error { @@ -441,7 +535,8 @@ func (a *Agent) DisableNodeMaintenance() error { // Monitor returns a channel which will receive streaming logs from the agent // Providing a non-nil stopCh can be used to close the connection and stop the -// log stream +// log stream. An empty string will be sent down the given channel when there's +// nothing left to stream, after which the caller should close the stopCh. func (a *Agent) Monitor(loglevel string, stopCh <-chan struct{}, q *QueryOptions) (chan string, error) { r := a.c.newRequest("GET", "/v1/agent/monitor") r.setQueryOptions(q) @@ -466,10 +561,61 @@ func (a *Agent) Monitor(loglevel string, stopCh <-chan struct{}, q *QueryOptions default: } if scanner.Scan() { - logCh <- scanner.Text() + // An empty string signals to the caller that + // the scan is done, so make sure we only emit + // that when the scanner says it's done, not if + // we happen to ingest an empty line. + if text := scanner.Text(); text != "" { + logCh <- text + } else { + logCh <- " " + } + } else { + logCh <- "" } } }() return logCh, nil } + +// UpdateACLToken updates the agent's "acl_token". See updateToken for more +// details. +func (c *Agent) UpdateACLToken(token string, q *WriteOptions) (*WriteMeta, error) { + return c.updateToken("acl_token", token, q) +} + +// UpdateACLAgentToken updates the agent's "acl_agent_token". See updateToken +// for more details. +func (c *Agent) UpdateACLAgentToken(token string, q *WriteOptions) (*WriteMeta, error) { + return c.updateToken("acl_agent_token", token, q) +} + +// UpdateACLAgentMasterToken updates the agent's "acl_agent_master_token". See +// updateToken for more details. +func (c *Agent) UpdateACLAgentMasterToken(token string, q *WriteOptions) (*WriteMeta, error) { + return c.updateToken("acl_agent_master_token", token, q) +} + +// UpdateACLReplicationToken updates the agent's "acl_replication_token". See +// updateToken for more details. +func (c *Agent) UpdateACLReplicationToken(token string, q *WriteOptions) (*WriteMeta, error) { + return c.updateToken("acl_replication_token", token, q) +} + +// updateToken can be used to update an agent's ACL token after the agent has +// started. The tokens are not persisted, so will need to be updated again if +// the agent is restarted. +func (c *Agent) updateToken(target, token string, q *WriteOptions) (*WriteMeta, error) { + r := c.c.newRequest("PUT", fmt.Sprintf("/v1/agent/token/%s", target)) + r.setWriteOptions(q) + r.obj = &AgentToken{Token: token} + rtt, resp, err := requireOK(c.c.doRequest(r)) + if err != nil { + return nil, err + } + resp.Body.Close() + + wm := &WriteMeta{RequestTime: rtt} + return wm, nil +} diff --git a/vendor/github.com/hashicorp/consul/api/api.go b/vendor/github.com/hashicorp/consul/api/api.go index 0a62b4f68..97a524b5e 100644 --- a/vendor/github.com/hashicorp/consul/api/api.go +++ b/vendor/github.com/hashicorp/consul/api/api.go @@ -446,6 +446,7 @@ func NewClient(config *Config) (*Client, error) { if len(parts) == 2 { switch parts[0] { case "http": + config.Scheme = "http" case "https": config.Scheme = "https" case "unix": @@ -462,10 +463,11 @@ func NewClient(config *Config) (*Client, error) { config.Address = parts[1] } - client := &Client{ - config: *config, + if config.Token == "" { + config.Token = defConfig.Token } - return client, nil + + return &Client{config: *config}, nil } // NewHttpClient returns an http client configured with the given Transport and TLS @@ -553,13 +555,20 @@ func durToMsec(dur time.Duration) string { // serverError is a string we look for to detect 500 errors. const serverError = "Unexpected response code: 500" -// IsServerError returns true for 500 errors from the Consul servers, these are -// usually retryable at a later time. -func IsServerError(err error) bool { +// IsRetryableError returns true for 500 errors from the Consul servers, and +// network connection errors. These are usually retryable at a later time. +// This applies to reads but NOT to writes. This may return true for errors +// on writes that may have still gone through, so do not use this to retry +// any write operations. +func IsRetryableError(err error) bool { if err == nil { return false } + if _, ok := err.(net.Error); ok { + return true + } + // TODO (slackpad) - Make a real error type here instead of using // a string check. return strings.Contains(err.Error(), serverError) @@ -652,7 +661,7 @@ func (c *Client) doRequest(r *request) (time.Duration, *http.Response, error) { } start := time.Now() resp, err := c.config.HttpClient.Do(req) - diff := time.Now().Sub(start) + diff := time.Since(start) return diff, resp, err } diff --git a/vendor/github.com/hashicorp/consul/api/coordinate.go b/vendor/github.com/hashicorp/consul/api/coordinate.go index ae8d16ee6..90214e392 100644 --- a/vendor/github.com/hashicorp/consul/api/coordinate.go +++ b/vendor/github.com/hashicorp/consul/api/coordinate.go @@ -6,8 +6,9 @@ import ( // CoordinateEntry represents a node and its associated network coordinate. type CoordinateEntry struct { - Node string - Coord *coordinate.Coordinate + Node string + Segment string + Coord *coordinate.Coordinate } // CoordinateDatacenterMap has the coordinates for servers in a given datacenter diff --git a/vendor/github.com/hashicorp/consul/api/kv.go b/vendor/github.com/hashicorp/consul/api/kv.go index f91bb50fc..97f515685 100644 --- a/vendor/github.com/hashicorp/consul/api/kv.go +++ b/vendor/github.com/hashicorp/consul/api/kv.go @@ -252,7 +252,7 @@ func (k *KV) put(key string, params map[string]string, body []byte, q *WriteOpti if _, err := io.Copy(&buf, resp.Body); err != nil { return false, nil, fmt.Errorf("Failed to read response: %v", err) } - res := strings.Contains(string(buf.Bytes()), "true") + res := strings.Contains(buf.String(), "true") return res, qm, nil } @@ -296,7 +296,7 @@ func (k *KV) deleteInternal(key string, params map[string]string, q *WriteOption if _, err := io.Copy(&buf, resp.Body); err != nil { return false, nil, fmt.Errorf("Failed to read response: %v", err) } - res := strings.Contains(string(buf.Bytes()), "true") + res := strings.Contains(buf.String(), "true") return res, qm, nil } @@ -353,19 +353,19 @@ type TxnResponse struct { // // Here's an example: // -// ops := KVTxnOps{ -// &KVTxnOp{ -// Verb: KVLock, -// Key: "test/lock", -// Session: "adf4238a-882b-9ddc-4a9d-5b6758e4159e", -// Value: []byte("hello"), -// }, -// &KVTxnOp{ -// Verb: KVGet, -// Key: "another/key", -// }, -// } -// ok, response, _, err := kv.Txn(&ops, nil) +// ops := KVTxnOps{ +// &KVTxnOp{ +// Verb: KVLock, +// Key: "test/lock", +// Session: "adf4238a-882b-9ddc-4a9d-5b6758e4159e", +// Value: []byte("hello"), +// }, +// &KVTxnOp{ +// Verb: KVGet, +// Key: "another/key", +// }, +// } +// ok, response, _, err := kv.Txn(&ops, nil) // // If there is a problem making the transaction request then an error will be // returned. Otherwise, the ok value will be true if the transaction succeeded diff --git a/vendor/github.com/hashicorp/consul/api/lock.go b/vendor/github.com/hashicorp/consul/api/lock.go index 466ef5fdf..41f72e7d2 100644 --- a/vendor/github.com/hashicorp/consul/api/lock.go +++ b/vendor/github.com/hashicorp/consul/api/lock.go @@ -180,7 +180,7 @@ WAIT: // Handle the one-shot mode. if l.opts.LockTryOnce && attempts > 0 { - elapsed := time.Now().Sub(start) + elapsed := time.Since(start) if elapsed > qOpts.WaitTime { return nil, nil } @@ -370,7 +370,7 @@ RETRY: // by doing retries. Note that we have to attempt the retry in a non- // blocking fashion so that we have a clean place to reset the retry // counter if service is restored. - if retries > 0 && IsServerError(err) { + if retries > 0 && IsRetryableError(err) { time.Sleep(l.opts.MonitorRetryTime) retries-- opts.WaitIndex = 0 diff --git a/vendor/github.com/hashicorp/consul/api/operator_area.go b/vendor/github.com/hashicorp/consul/api/operator_area.go index 7b0e461e9..a630b694c 100644 --- a/vendor/github.com/hashicorp/consul/api/operator_area.go +++ b/vendor/github.com/hashicorp/consul/api/operator_area.go @@ -25,6 +25,10 @@ type Area struct { // RetryJoin specifies the address of Consul servers to join to, such as // an IPs or hostnames with an optional port number. This is optional. RetryJoin []string + + // UseTLS specifies whether gossip over this area should be encrypted with TLS + // if possible. + UseTLS bool } // AreaJoinResponse is returned when a join occurs and gives the result for each @@ -100,6 +104,27 @@ func (op *Operator) AreaCreate(area *Area, q *WriteOptions) (string, *WriteMeta, return out.ID, wm, nil } +// AreaUpdate will update the configuration of the network area with the given ID. +func (op *Operator) AreaUpdate(areaID string, area *Area, q *WriteOptions) (string, *WriteMeta, error) { + r := op.c.newRequest("PUT", "/v1/operator/area/"+areaID) + r.setWriteOptions(q) + r.obj = area + rtt, resp, err := requireOK(op.c.doRequest(r)) + if err != nil { + return "", nil, err + } + defer resp.Body.Close() + + wm := &WriteMeta{} + wm.RequestTime = rtt + + var out struct{ ID string } + if err := decodeBody(resp, &out); err != nil { + return "", nil, err + } + return out.ID, wm, nil +} + // AreaGet returns a single network area. func (op *Operator) AreaGet(areaID string, q *QueryOptions) ([]*Area, *QueryMeta, error) { var out []*Area diff --git a/vendor/github.com/hashicorp/consul/api/operator_autopilot.go b/vendor/github.com/hashicorp/consul/api/operator_autopilot.go index 0fa9d1604..b179406dc 100644 --- a/vendor/github.com/hashicorp/consul/api/operator_autopilot.go +++ b/vendor/github.com/hashicorp/consul/api/operator_autopilot.go @@ -196,7 +196,7 @@ func (op *Operator) AutopilotCASConfiguration(conf *AutopilotConfiguration, q *W if _, err := io.Copy(&buf, resp.Body); err != nil { return false, fmt.Errorf("Failed to read response: %v", err) } - res := strings.Contains(string(buf.Bytes()), "true") + res := strings.Contains(buf.String(), "true") return res, nil } diff --git a/vendor/github.com/hashicorp/consul/api/operator_keyring.go b/vendor/github.com/hashicorp/consul/api/operator_keyring.go index 4f91c3543..6b614296c 100644 --- a/vendor/github.com/hashicorp/consul/api/operator_keyring.go +++ b/vendor/github.com/hashicorp/consul/api/operator_keyring.go @@ -13,6 +13,9 @@ type KeyringResponse struct { // The datacenter name this request corresponds to Datacenter string + // Segment has the network segment this request corresponds to. + Segment string + // A map of the encryption keys to the number of nodes they're installed on Keys map[string]int diff --git a/vendor/github.com/hashicorp/consul/api/operator_raft.go b/vendor/github.com/hashicorp/consul/api/operator_raft.go index 5f3c25b13..a9844df2d 100644 --- a/vendor/github.com/hashicorp/consul/api/operator_raft.go +++ b/vendor/github.com/hashicorp/consul/api/operator_raft.go @@ -17,6 +17,9 @@ type RaftServer struct { // Leader is true if this server is the current cluster leader. Leader bool + // Protocol version is the raft protocol version used by the server + ProtocolVersion string + // Voter is true if this server has a vote in the cluster. This might // be false if the server is staging and still coming online, or if // it's a non-voting server, which will be added in a future release of @@ -24,7 +27,7 @@ type RaftServer struct { Voter bool } -// RaftConfigration is returned when querying for the current Raft configuration. +// RaftConfiguration is returned when querying for the current Raft configuration. type RaftConfiguration struct { // Servers has the list of servers in the Raft configuration. Servers []*RaftServer diff --git a/vendor/github.com/hashicorp/consul/api/operator_segment.go b/vendor/github.com/hashicorp/consul/api/operator_segment.go new file mode 100644 index 000000000..92b05d3c0 --- /dev/null +++ b/vendor/github.com/hashicorp/consul/api/operator_segment.go @@ -0,0 +1,11 @@ +package api + +// SegmentList returns all the available LAN segments. +func (op *Operator) SegmentList(q *QueryOptions) ([]string, *QueryMeta, error) { + var out []string + qm, err := op.c.query("/v1/operator/segment", &out, q) + if err != nil { + return nil, nil, err + } + return out, qm, nil +} diff --git a/vendor/github.com/hashicorp/consul/api/semaphore.go b/vendor/github.com/hashicorp/consul/api/semaphore.go index 9ddbdc49e..d0c574177 100644 --- a/vendor/github.com/hashicorp/consul/api/semaphore.go +++ b/vendor/github.com/hashicorp/consul/api/semaphore.go @@ -198,7 +198,7 @@ WAIT: // Handle the one-shot mode. if s.opts.SemaphoreTryOnce && attempts > 0 { - elapsed := time.Now().Sub(start) + elapsed := time.Since(start) if elapsed > qOpts.WaitTime { return nil, nil } @@ -492,7 +492,7 @@ RETRY: // by doing retries. Note that we have to attempt the retry in a non- // blocking fashion so that we have a clean place to reset the retry // counter if service is restored. - if retries > 0 && IsServerError(err) { + if retries > 0 && IsRetryableError(err) { time.Sleep(s.opts.MonitorRetryTime) retries-- opts.WaitIndex = 0 diff --git a/vendor/github.com/hashicorp/consul/lib/freeport/freeport.go b/vendor/github.com/hashicorp/consul/lib/freeport/freeport.go new file mode 100644 index 000000000..882998314 --- /dev/null +++ b/vendor/github.com/hashicorp/consul/lib/freeport/freeport.go @@ -0,0 +1,139 @@ +// Package freeport provides a helper for allocating free ports across multiple +// processes on the same machine. +package freeport + +import ( + "fmt" + "math/rand" + "net" + "sync" + "time" + + "github.com/mitchellh/go-testing-interface" +) + +const ( + // blockSize is the size of the allocated port block. ports are given out + // consecutively from that block with roll-over for the lifetime of the + // application/test run. + blockSize = 500 + + // maxBlocks is the number of available port blocks. + // lowPort + maxBlocks * blockSize must be less than 65535. + maxBlocks = 30 + + // lowPort is the lowest port number that should be used. + lowPort = 10000 + + // attempts is how often we try to allocate a port block + // before giving up. + attempts = 10 +) + +var ( + // firstPort is the first port of the allocated block. + firstPort int + + // lockLn is the system-wide mutex for the port block. + lockLn net.Listener + + // mu guards nextPort + mu sync.Mutex + + // once is used to do the initialization on the first call to retrieve free + // ports + once sync.Once + + // port is the last allocated port. + port int +) + +// initialize is used to initialize freeport. +func initialize() { + if lowPort+maxBlocks*blockSize > 65535 { + panic("freeport: block size too big or too many blocks requested") + } + + rand.Seed(time.Now().UnixNano()) + firstPort, lockLn = alloc() +} + +// alloc reserves a port block for exclusive use for the lifetime of the +// application. lockLn serves as a system-wide mutex for the port block and is +// implemented as a TCP listener which is bound to the firstPort and which will +// be automatically released when the application terminates. +func alloc() (int, net.Listener) { + for i := 0; i < attempts; i++ { + block := int(rand.Int31n(int32(maxBlocks))) + firstPort := lowPort + block*blockSize + ln, err := net.ListenTCP("tcp", tcpAddr("127.0.0.1", firstPort)) + if err != nil { + continue + } + // log.Printf("[DEBUG] freeport: allocated port block %d (%d-%d)", block, firstPort, firstPort+blockSize-1) + return firstPort, ln + } + panic("freeport: cannot allocate port block") +} + +func tcpAddr(ip string, port int) *net.TCPAddr { + return &net.TCPAddr{IP: net.ParseIP(ip), Port: port} +} + +// Get wraps the Free function and panics on any failure retrieving ports. +func Get(n int) (ports []int) { + ports, err := Free(n) + if err != nil { + panic(err) + } + + return ports +} + +// GetT is suitable for use when retrieving unused ports in tests. If there is +// an error retrieving free ports, the test will be failed. +func GetT(t testing.T, n int) (ports []int) { + ports, err := Free(n) + if err != nil { + t.Fatalf("Failed retrieving free port: %v", err) + } + + return ports +} + +// Free returns a list of free ports from the allocated port block. It is safe +// to call this method concurrently. Ports have been tested to be available on +// 127.0.0.1 TCP but there is no guarantee that they will remain free in the +// future. +func Free(n int) (ports []int, err error) { + mu.Lock() + defer mu.Unlock() + + if n > blockSize-1 { + return nil, fmt.Errorf("freeport: block size too small") + } + + // Reserve a port block + once.Do(initialize) + + for len(ports) < n { + port++ + + // roll-over the port + if port < firstPort+1 || port >= firstPort+blockSize { + port = firstPort + 1 + } + + // if the port is in use then skip it + ln, err := net.ListenTCP("tcp", tcpAddr("127.0.0.1", port)) + if err != nil { + // log.Println("[DEBUG] freeport: port already in use: ", port) + continue + } + ln.Close() + + ports = append(ports, port) + } + // log.Println("[DEBUG] freeport: free ports:", ports) + return ports, nil +} diff --git a/vendor/github.com/hashicorp/consul/testutil/server.go b/vendor/github.com/hashicorp/consul/testutil/server.go index 969d06a58..4993f13cb 100644 --- a/vendor/github.com/hashicorp/consul/testutil/server.go +++ b/vendor/github.com/hashicorp/consul/testutil/server.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "github.com/hashicorp/consul/lib/freeport" "github.com/hashicorp/consul/testutil/retry" "github.com/hashicorp/go-cleanhttp" "github.com/hashicorp/go-uuid" @@ -47,9 +48,6 @@ type TestPortConfig struct { SerfLan int `json:"serf_lan,omitempty"` SerfWan int `json:"serf_wan,omitempty"` Server int `json:"server,omitempty"` - - // Deprecated - RPC int `json:"rpc,omitempty"` } // TestAddressConfig contains the bind addresses for various @@ -58,6 +56,14 @@ type TestAddressConfig struct { HTTP string `json:"http,omitempty"` } +// TestNetworkSegment contains the configuration for a network segment. +type TestNetworkSegment struct { + Name string `json:"name"` + Bind string `json:"bind"` + Port int `json:"port"` + Advertise string `json:"advertise"` +} + // TestServerConfig is the main server configuration struct. type TestServerConfig struct { NodeName string `json:"node_name"` @@ -68,6 +74,7 @@ type TestServerConfig struct { Server bool `json:"server,omitempty"` DataDir string `json:"data_dir,omitempty"` Datacenter string `json:"datacenter,omitempty"` + Segments []TestNetworkSegment `json:"segments"` DisableCheckpoint bool `json:"disable_update_check"` LogLevel string `json:"log_level,omitempty"` Bind string `json:"bind_addr,omitempty"` @@ -104,8 +111,9 @@ func defaultServerConfig() *TestServerConfig { panic(err) } + ports := freeport.Get(6) return &TestServerConfig{ - NodeName: fmt.Sprintf("node%d", randomPort()), + NodeName: "node-" + nodeID, NodeID: nodeID, DisableCheckpoint: true, Performance: &TestPerformanceConfig{ @@ -117,28 +125,17 @@ func defaultServerConfig() *TestServerConfig { Bind: "127.0.0.1", Addresses: &TestAddressConfig{}, Ports: &TestPortConfig{ - DNS: randomPort(), - HTTP: randomPort(), - HTTPS: randomPort(), - SerfLan: randomPort(), - SerfWan: randomPort(), - Server: randomPort(), - RPC: randomPort(), + DNS: ports[0], + HTTP: ports[1], + HTTPS: ports[2], + SerfLan: ports[3], + SerfWan: ports[4], + Server: ports[5], }, ReadyTimeout: 10 * time.Second, } } -// randomPort asks the kernel for a random port to use. -func randomPort() int { - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - panic(err) - } - defer l.Close() - return l.Addr().(*net.TCPAddr).Port -} - // TestService is used to serialize a service definition. type TestService struct { ID string `json:",omitempty"` @@ -191,15 +188,7 @@ func NewTestServerConfig(cb ServerConfigCallback) (*TestServer, error) { // configuring or starting the server, the server will NOT be running when the // function returns (thus you do not need to stop it). func NewTestServerConfigT(t *testing.T, cb ServerConfigCallback) (*TestServer, error) { - var server *TestServer - retry.Run(t, func(r *retry.R) { - var err error - server, err = newTestServerConfigT(t, cb) - if err != nil { - r.Fatalf("failed starting test server: %v", err) - } - }) - return server, nil + return newTestServerConfigT(t, cb) } // newTestServerConfigT is the internal helper for NewTestServerConfigT. diff --git a/vendor/github.com/hashicorp/consul/testutil/server_methods.go b/vendor/github.com/hashicorp/consul/testutil/server_methods.go index 8f4b067ad..dec512054 100644 --- a/vendor/github.com/hashicorp/consul/testutil/server_methods.go +++ b/vendor/github.com/hashicorp/consul/testutil/server_methods.go @@ -25,13 +25,13 @@ const ( // JoinLAN is used to join local datacenters together. func (s *TestServer) JoinLAN(t *testing.T, addr string) { - resp := s.get(t, "/v1/agent/join/"+addr) + resp := s.put(t, "/v1/agent/join/"+addr, nil) defer resp.Body.Close() } // JoinWAN is used to join remote datacenters together. func (s *TestServer) JoinWAN(t *testing.T, addr string) { - resp := s.get(t, "/v1/agent/join/"+addr+"?wan=1") + resp := s.put(t, "/v1/agent/join/"+addr+"?wan=1", nil) resp.Body.Close() } diff --git a/vendor/vendor.json b/vendor/vendor.json index 4a84962f8..0dafed574 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -1421,29 +1421,35 @@ "revisionTime": "2016-11-07T20:49:10Z" }, { - "checksumSHA1": "IYuLg7xUzsf/P9rMpdEh1n9rbIY=", + "checksumSHA1": "4hc6jGp9/1m7dp5ACRcGnspjO5E=", "comment": "v0.6.3-28-g3215b87", "path": "github.com/hashicorp/consul/api", - "revision": "b79d951ced8c5f18fe73d35b2806f3435e40cd64", - "revisionTime": "2017-07-20T03:19:26Z", - "version": "v0.9.0", - "versionExact": "v0.9.0" + "revision": "610f3c86a089817b5bd5729a3b8c2db33a9ae2b0", + "revisionTime": "2017-10-26T17:59:57Z", + "version": "master", + "versionExact": "master" }, { - "checksumSHA1": "++0PVBxbpylmllyCxSa7cdc6dDc=", + "checksumSHA1": "hDJiPli3EEGJE4vAezMi05oOC7o=", + "path": "github.com/hashicorp/consul/lib/freeport", + "revision": "610f3c86a089817b5bd5729a3b8c2db33a9ae2b0", + "revisionTime": "2017-10-26T17:59:57Z" + }, + { + "checksumSHA1": "gjN5oVYQR1K7hCbUq2ED4KBWuys=", "path": "github.com/hashicorp/consul/testutil", - "revision": "b79d951ced8c5f18fe73d35b2806f3435e40cd64", - "revisionTime": "2017-07-20T03:19:26Z", - "version": "v0.9.0", - "versionExact": "v0.9.0" + "revision": "610f3c86a089817b5bd5729a3b8c2db33a9ae2b0", + "revisionTime": "2017-10-26T17:59:57Z", + "version": "master", + "versionExact": "master" }, { "checksumSHA1": "J8TTDc84MvAyXE/FrfgS+xc/b6s=", "path": "github.com/hashicorp/consul/testutil/retry", - "revision": "b79d951ced8c5f18fe73d35b2806f3435e40cd64", - "revisionTime": "2017-07-20T03:19:26Z", - "version": "v0.9.0", - "versionExact": "v0.9.0" + "revision": "610f3c86a089817b5bd5729a3b8c2db33a9ae2b0", + "revisionTime": "2017-10-26T17:59:57Z", + "version": "master", + "versionExact": "master" }, { "checksumSHA1": "cdOCt0Yb+hdErz8NAQqayxPmRsY=", From cf54ca3b0f7749c293e4f6673b0b553043071b2c Mon Sep 17 00:00:00 2001 From: James Bardin Date: Sat, 28 Oct 2017 19:45:10 -0400 Subject: [PATCH 2/2] update tests for new consul packages Reuse the running consul server for all tests. Update the lostLockConnection package, since the api client should no longer lose a lock immediately on network errors. --- backend/remote-state/consul/backend_test.go | 41 +++++++------- backend/remote-state/consul/client_test.go | 62 +++++++-------------- 2 files changed, 42 insertions(+), 61 deletions(-) diff --git a/backend/remote-state/consul/backend_test.go b/backend/remote-state/consul/backend_test.go index 6beda5f5b..649fd707e 100644 --- a/backend/remote-state/consul/backend_test.go +++ b/backend/remote-state/consul/backend_test.go @@ -15,14 +15,28 @@ func TestBackend_impl(t *testing.T) { var _ backend.Backend = new(Backend) } -func newConsulTestServer(t *testing.T) *testutil.TestServer { - skip := os.Getenv("TF_ACC") == "" && os.Getenv("TF_CONSUL_TEST") == "" - if skip { - t.Log("consul server tests require setting TF_ACC or TF_CONSUL_TEST") - t.Skip() +var srv *testutil.TestServer + +func TestMain(m *testing.M) { + if os.Getenv("TF_ACC") == "" && os.Getenv("TF_CONSUL_TEST") == "" { + fmt.Println("consul server tests require setting TF_ACC or TF_CONSUL_TEST") + return } - srv, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + var err error + srv, err = newConsulTestServer() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + rc := m.Run() + srv.Stop() + os.Exit(rc) +} + +func newConsulTestServer() (*testutil.TestServer, error) { + srv, err := testutil.NewTestServerConfig(func(c *testutil.TestServerConfig) { c.LogLevel = "warn" if !testing.Verbose() { @@ -31,17 +45,10 @@ func newConsulTestServer(t *testing.T) *testutil.TestServer { } }) - if err != nil { - t.Fatal(err) - } - - return srv + return srv, err } func TestBackend(t *testing.T) { - srv := newConsulTestServer(t) - defer srv.Stop() - path := fmt.Sprintf("tf-unit/%s", time.Now().String()) // Get the backend. We need two to test locking. @@ -60,9 +67,6 @@ func TestBackend(t *testing.T) { } func TestBackend_lockDisabled(t *testing.T) { - srv := newConsulTestServer(t) - defer srv.Stop() - path := fmt.Sprintf("tf-unit/%s", time.Now().String()) // Get the backend. We need two to test locking. @@ -83,9 +87,6 @@ func TestBackend_lockDisabled(t *testing.T) { } func TestBackend_gzip(t *testing.T) { - srv := newConsulTestServer(t) - defer srv.Stop() - // Get the backend b := backend.TestBackendConfig(t, New(), map[string]interface{}{ "address": srv.HTTPAddr, diff --git a/backend/remote-state/consul/client_test.go b/backend/remote-state/consul/client_test.go index ef3c5b187..b2c95c527 100644 --- a/backend/remote-state/consul/client_test.go +++ b/backend/remote-state/consul/client_test.go @@ -19,9 +19,6 @@ func TestRemoteClient_impl(t *testing.T) { } func TestRemoteClient(t *testing.T) { - srv := newConsulTestServer(t) - defer srv.Stop() - // Get the backend b := backend.TestBackendConfig(t, New(), map[string]interface{}{ "address": srv.HTTPAddr, @@ -40,9 +37,6 @@ func TestRemoteClient(t *testing.T) { // test the gzip functionality of the client func TestRemoteClient_gzipUpgrade(t *testing.T) { - srv := newConsulTestServer(t) - defer srv.Stop() - statePath := fmt.Sprintf("tf-unit/%s", time.Now().String()) // Get the backend @@ -78,9 +72,6 @@ func TestRemoteClient_gzipUpgrade(t *testing.T) { } func TestConsul_stateLock(t *testing.T) { - srv := newConsulTestServer(t) - defer srv.Stop() - path := fmt.Sprintf("tf-unit/%s", time.Now().String()) // create 2 instances to get 2 remote.Clients @@ -104,9 +95,6 @@ func TestConsul_stateLock(t *testing.T) { } func TestConsul_destroyLock(t *testing.T) { - srv := newConsulTestServer(t) - defer srv.Stop() - // Get the backend b := backend.TestBackendConfig(t, New(), map[string]interface{}{ "address": srv.HTTPAddr, @@ -144,9 +132,6 @@ func TestConsul_destroyLock(t *testing.T) { } func TestConsul_lostLock(t *testing.T) { - srv := newConsulTestServer(t) - defer srv.Stop() - path := fmt.Sprintf("tf-unit/%s", time.Now().String()) // create 2 instances to get 2 remote.Clients @@ -194,9 +179,6 @@ func TestConsul_lostLock(t *testing.T) { } func TestConsul_lostLockConnection(t *testing.T) { - srv := newConsulTestServer(t) - defer srv.Stop() - // create an "unreliable" network by closing all the consul client's // network connections conns := &unreliableConns{} @@ -225,31 +207,17 @@ func TestConsul_lostLockConnection(t *testing.T) { t.Fatal(err) } - // set a callback to know when the monitor loop re-connects - dialed := make(chan struct{}) - conns.dialCallback = func() { - close(dialed) - conns.dialCallback = nil + // kill the connection a few times + for i := 0; i < 3; i++ { + dialed := conns.dialedDone() + // kill any open connections + conns.Kill() + // wait for a new connection to be dialed, and kill it again + <-dialed } - // kill any open connections - // once the consul client is fixed, we should loop over this a few time to - // be sure, since we can't hook into the client's internal lock monitor - // loop. - conns.Kill() - // wait for a new connection to be dialed, and kill it again - <-dialed - conns.Kill() - - // since the lock monitor loop is hidden in the consul api client, we can - // only wait a bit to make sure we were notified of the failure - time.Sleep(time.Second) - - // once the consul client can reconnect properly, there will no longer be - // an error here - //if err := s.Unlock(id); err != nil { - if err := s.Unlock(id); err != lostLockErr { - t.Fatalf("expected lost lock error, got %v", err) + if err := s.Unlock(id); err != nil { + t.Fatal("unlock error:", err) } } @@ -278,6 +246,18 @@ func (u *unreliableConns) DialContext(ctx context.Context, netw, addr string) (n return conn, nil } +func (u *unreliableConns) dialedDone() chan struct{} { + u.Lock() + defer u.Unlock() + dialed := make(chan struct{}) + u.dialCallback = func() { + defer close(dialed) + u.dialCallback = nil + } + + return dialed +} + // Kill these with a deadline, just to make sure we don't end up with any EOFs // that get ignored. func (u *unreliableConns) Kill() {