core: close provider/provisioner connections

Currently Terraform is leaking goroutines and with that memory. I know
strictly speaking this maybe isn’t a real concern for Terraform as it’s
mostly used as a short running command line executable.

But there are a few of us out there that are using Terraform in some
long running processes and then this starts to become a problem.

Next to that it’s of course good programming practise to clean up
resources when they're not needed anymore. So even for the standard
command line use case, this seems an improvement in resource management.

Personally I see no downsides as the primary connection to the plugin
is kept alive (the plugin is not killed) and only unused connections
that will never be used again are closed to free up any related
goroutines and memory.
This commit is contained in:
Sander van Harmelen 2015-06-19 21:52:50 +02:00
parent 7850bb39d5
commit 0b1dbf31a3
21 changed files with 601 additions and 46 deletions

View File

@ -3,9 +3,9 @@ package rpc
import ( import (
"encoding/binary" "encoding/binary"
"fmt" "fmt"
"math/rand"
"net" "net"
"sync" "sync"
"sync/atomic"
"time" "time"
"github.com/hashicorp/yamux" "github.com/hashicorp/yamux"
@ -17,7 +17,6 @@ import (
// or accept a connection from, and the broker handles the details of // or accept a connection from, and the broker handles the details of
// holding these channels open while they're being negotiated. // holding these channels open while they're being negotiated.
type muxBroker struct { type muxBroker struct {
nextId uint32
session *yamux.Session session *yamux.Session
streams map[uint32]*muxBrokerPending streams map[uint32]*muxBrokerPending
@ -95,9 +94,12 @@ func (m *muxBroker) Dial(id uint32) (net.Conn, error) {
return stream, nil return stream, nil
} }
// NextId returns a unique ID to use next. // NextId returns a unique ID to use next. There is no need for seeding the
// rand package as the returned ID's aren't stored or used anywhere outside
// the current runtime. So it's perfectly fine to get the same pseudo-random
// numbers each time terraform is running.
func (m *muxBroker) NextId() uint32 { func (m *muxBroker) NextId() uint32 {
return atomic.AddUint32(&m.nextId, 1) return rand.Uint32()
} }
// Run starts the brokering and should be executed in a goroutine, since it // Run starts the brokering and should be executed in a goroutine, since it

View File

@ -174,6 +174,10 @@ func (p *ResourceProvider) Resources() []terraform.ResourceType {
return result return result
} }
func (p *ResourceProvider) Close() error {
return p.Client.Close()
}
// ResourceProviderServer is a net/rpc compatible structure for serving // ResourceProviderServer is a net/rpc compatible structure for serving
// a ResourceProvider. This should not be used directly. // a ResourceProvider. This should not be used directly.
type ResourceProviderServer struct { type ResourceProviderServer struct {

View File

@ -488,3 +488,31 @@ func TestResourceProvider_validateResource_warns(t *testing.T) {
t.Fatalf("bad: %#v", w) t.Fatalf("bad: %#v", w)
} }
} }
func TestResourceProvider_close(t *testing.T) {
client, _ := testNewClientServer(t)
defer client.Close()
provider, err := client.ResourceProvider()
if err != nil {
t.Fatalf("err: %s", err)
}
var p interface{}
p = provider
pCloser, ok := p.(terraform.ResourceProviderCloser)
if !ok {
t.Fatal("should be a ResourceProviderCloser")
}
if err := pCloser.Close(); err != nil {
t.Fatalf("failed to close provider: %s", err)
}
// The connection should be closed now, so if we to make a
// new call we should get an error.
err = provider.Configure(&terraform.ResourceConfig{})
if err == nil {
t.Fatal("should have error")
}
}

View File

@ -63,6 +63,10 @@ func (p *ResourceProvisioner) Apply(
return err return err
} }
func (p *ResourceProvisioner) Close() error {
return p.Client.Close()
}
type ResourceProvisionerValidateArgs struct { type ResourceProvisionerValidateArgs struct {
Config *terraform.ResourceConfig Config *terraform.ResourceConfig
} }

View File

@ -132,3 +132,34 @@ func TestResourceProvisioner_validate_warns(t *testing.T) {
t.Fatalf("bad: %#v", w) t.Fatalf("bad: %#v", w)
} }
} }
func TestResourceProvisioner_close(t *testing.T) {
client, _ := testNewClientServer(t)
defer client.Close()
provisioner, err := client.ResourceProvisioner()
if err != nil {
t.Fatalf("err: %s", err)
}
var p interface{}
p = provisioner
pCloser, ok := p.(terraform.ResourceProvisionerCloser)
if !ok {
t.Fatal("should be a ResourceProvisionerCloser")
}
if err := pCloser.Close(); err != nil {
t.Fatalf("failed to close provisioner: %s", err)
}
// The connection should be closed now, so if we to make a
// new call we should get an error.
o := &terraform.MockUIOutput{}
s := &terraform.InstanceState{}
c := &terraform.ResourceConfig{}
err = provisioner.Apply(o, s, c)
if err == nil {
t.Fatal("should have error")
}
}

View File

@ -28,6 +28,9 @@ type EvalContext interface {
// initialized) or returns nil if the provider isn't initialized. // initialized) or returns nil if the provider isn't initialized.
Provider(string) ResourceProvider Provider(string) ResourceProvider
// CloseProvider closes provider connections that aren't needed anymore.
CloseProvider(string) error
// ConfigureProvider configures the provider with the given // ConfigureProvider configures the provider with the given
// configuration. This is a separate context call because this call // configuration. This is a separate context call because this call
// is used to store the provider configuration for inheritance lookups // is used to store the provider configuration for inheritance lookups
@ -51,6 +54,10 @@ type EvalContext interface {
// initialized) or returns nil if the provisioner isn't initialized. // initialized) or returns nil if the provisioner isn't initialized.
Provisioner(string) ResourceProvisioner Provisioner(string) ResourceProvisioner
// CloseProvisioner closes provisioner connections that aren't needed
// anymore.
CloseProvisioner(string) error
// Interpolate takes the given raw configuration and completes // Interpolate takes the given raw configuration and completes
// the interpolations, returning the processed ResourceConfig. // the interpolations, returning the processed ResourceConfig.
// //

View File

@ -114,6 +114,28 @@ func (ctx *BuiltinEvalContext) Provider(n string) ResourceProvider {
return ctx.ProviderCache[PathCacheKey(providerPath)] return ctx.ProviderCache[PathCacheKey(providerPath)]
} }
func (ctx *BuiltinEvalContext) CloseProvider(n string) error {
ctx.once.Do(ctx.init)
ctx.ProviderLock.Lock()
defer ctx.ProviderLock.Unlock()
providerPath := make([]string, len(ctx.Path())+1)
copy(providerPath, ctx.Path())
providerPath[len(providerPath)-1] = n
var provider interface{}
provider = ctx.ProviderCache[PathCacheKey(providerPath)]
if provider != nil {
if p, ok := provider.(ResourceProviderCloser); ok {
delete(ctx.ProviderCache, PathCacheKey(providerPath))
return p.Close()
}
}
return nil
}
func (ctx *BuiltinEvalContext) ConfigureProvider( func (ctx *BuiltinEvalContext) ConfigureProvider(
n string, cfg *ResourceConfig) error { n string, cfg *ResourceConfig) error {
p := ctx.Provider(n) p := ctx.Provider(n)
@ -222,6 +244,28 @@ func (ctx *BuiltinEvalContext) Provisioner(n string) ResourceProvisioner {
return ctx.ProvisionerCache[PathCacheKey(provPath)] return ctx.ProvisionerCache[PathCacheKey(provPath)]
} }
func (ctx *BuiltinEvalContext) CloseProvisioner(n string) error {
ctx.once.Do(ctx.init)
ctx.ProvisionerLock.Lock()
defer ctx.ProvisionerLock.Unlock()
provPath := make([]string, len(ctx.Path())+1)
copy(provPath, ctx.Path())
provPath[len(provPath)-1] = n
var prov interface{}
prov = ctx.ProvisionerCache[PathCacheKey(provPath)]
if prov != nil {
if p, ok := prov.(ResourceProvisionerCloser); ok {
delete(ctx.ProvisionerCache, PathCacheKey(provPath))
return p.Close()
}
}
return nil
}
func (ctx *BuiltinEvalContext) Interpolate( func (ctx *BuiltinEvalContext) Interpolate(
cfg *config.RawConfig, r *Resource) (*ResourceConfig, error) { cfg *config.RawConfig, r *Resource) (*ResourceConfig, error) {
if cfg != nil { if cfg != nil {

View File

@ -25,6 +25,10 @@ type MockEvalContext struct {
ProviderName string ProviderName string
ProviderProvider ResourceProvider ProviderProvider ResourceProvider
CloseProviderCalled bool
CloseProviderName string
CloseProviderProvider ResourceProvider
ProviderInputCalled bool ProviderInputCalled bool
ProviderInputName string ProviderInputName string
ProviderInputConfig map[string]interface{} ProviderInputConfig map[string]interface{}
@ -55,6 +59,10 @@ type MockEvalContext struct {
ProvisionerName string ProvisionerName string
ProvisionerProvisioner ResourceProvisioner ProvisionerProvisioner ResourceProvisioner
CloseProvisionerCalled bool
CloseProvisionerName string
CloseProvisionerProvisioner ResourceProvisioner
InterpolateCalled bool InterpolateCalled bool
InterpolateConfig *config.RawConfig InterpolateConfig *config.RawConfig
InterpolateResource *Resource InterpolateResource *Resource
@ -105,6 +113,12 @@ func (c *MockEvalContext) Provider(n string) ResourceProvider {
return c.ProviderProvider return c.ProviderProvider
} }
func (c *MockEvalContext) CloseProvider(n string) error {
c.CloseProviderCalled = true
c.CloseProviderName = n
return nil
}
func (c *MockEvalContext) ConfigureProvider(n string, cfg *ResourceConfig) error { func (c *MockEvalContext) ConfigureProvider(n string, cfg *ResourceConfig) error {
c.ConfigureProviderCalled = true c.ConfigureProviderCalled = true
c.ConfigureProviderName = n c.ConfigureProviderName = n
@ -150,6 +164,12 @@ func (c *MockEvalContext) Provisioner(n string) ResourceProvisioner {
return c.ProvisionerProvisioner return c.ProvisionerProvisioner
} }
func (c *MockEvalContext) CloseProvisioner(n string) error {
c.CloseProvisionerCalled = true
c.CloseProvisionerName = n
return nil
}
func (c *MockEvalContext) Interpolate( func (c *MockEvalContext) Interpolate(
config *config.RawConfig, resource *Resource) (*ResourceConfig, error) { config *config.RawConfig, resource *Resource) (*ResourceConfig, error) {
c.InterpolateCalled = true c.InterpolateCalled = true

View File

@ -71,6 +71,17 @@ func (n *EvalInitProvider) Eval(ctx EvalContext) (interface{}, error) {
return ctx.InitProvider(n.Name) return ctx.InitProvider(n.Name)
} }
// EvalCloseProvider is an EvalNode implementation that closes provider
// connections that aren't needed anymore.
type EvalCloseProvider struct {
Name string
}
func (n *EvalCloseProvider) Eval(ctx EvalContext) (interface{}, error) {
ctx.CloseProvider(n.Name)
return nil, nil
}
// EvalGetProvider is an EvalNode implementation that retrieves an already // EvalGetProvider is an EvalNode implementation that retrieves an already
// initialized provider instance for the given name. // initialized provider instance for the given name.
type EvalGetProvider struct { type EvalGetProvider struct {

View File

@ -112,6 +112,22 @@ func TestEvalInitProvider(t *testing.T) {
} }
} }
func TestEvalCloseProvider(t *testing.T) {
n := &EvalCloseProvider{Name: "foo"}
provider := &MockResourceProvider{}
ctx := &MockEvalContext{CloseProviderProvider: provider}
if _, err := n.Eval(ctx); err != nil {
t.Fatalf("err: %s", err)
}
if !ctx.CloseProviderCalled {
t.Fatal("should be called")
}
if ctx.CloseProviderName != "foo" {
t.Fatalf("bad: %#v", ctx.CloseProviderName)
}
}
func TestEvalGetProvider_impl(t *testing.T) { func TestEvalGetProvider_impl(t *testing.T) {
var _ EvalNode = new(EvalGetProvider) var _ EvalNode = new(EvalGetProvider)
} }

View File

@ -15,6 +15,17 @@ func (n *EvalInitProvisioner) Eval(ctx EvalContext) (interface{}, error) {
return ctx.InitProvisioner(n.Name) return ctx.InitProvisioner(n.Name)
} }
// EvalCloseProvisioner is an EvalNode implementation that closes provisioner
// connections that aren't needed anymore.
type EvalCloseProvisioner struct {
Name string
}
func (n *EvalCloseProvisioner) Eval(ctx EvalContext) (interface{}, error) {
ctx.CloseProvisioner(n.Name)
return nil, nil
}
// EvalGetProvisioner is an EvalNode implementation that retrieves an already // EvalGetProvisioner is an EvalNode implementation that retrieves an already
// initialized provisioner instance for the given name. // initialized provisioner instance for the given name.
type EvalGetProvisioner struct { type EvalGetProvisioner struct {

View File

@ -24,6 +24,22 @@ func TestEvalInitProvisioner(t *testing.T) {
} }
} }
func TestEvalCloseProvisioner(t *testing.T) {
n := &EvalCloseProvisioner{Name: "foo"}
provisioner := &MockResourceProvisioner{}
ctx := &MockEvalContext{CloseProvisionerProvisioner: provisioner}
if _, err := n.Eval(ctx); err != nil {
t.Fatalf("err: %s", err)
}
if !ctx.CloseProvisionerCalled {
t.Fatal("should be called")
}
if ctx.CloseProvisionerName != "foo" {
t.Fatalf("bad: %#v", ctx.CloseProvisionerName)
}
}
func TestEvalGetProvisioner_impl(t *testing.T) { func TestEvalGetProvisioner_impl(t *testing.T) {
var _ EvalNode = new(EvalGetProvisioner) var _ EvalNode = new(EvalGetProvisioner)
} }

View File

@ -72,3 +72,9 @@ func ProviderEvalTree(n string, config *config.RawConfig) EvalNode {
return &EvalSequence{Nodes: seq} return &EvalSequence{Nodes: seq}
} }
// CloseProviderEvalTree returns the evaluation tree for closing
// provider connections that aren't needed anymore.
func CloseProviderEvalTree(n string) EvalNode {
return &EvalCloseProvider{Name: n}
}

View File

@ -116,12 +116,14 @@ func (b *BuiltinGraphBuilder) Steps(path []string) []GraphTransformer {
// Provider-related transformations // Provider-related transformations
&MissingProviderTransformer{Providers: b.Providers}, &MissingProviderTransformer{Providers: b.Providers},
&ProviderTransformer{}, &ProviderTransformer{},
&CloseProviderTransformer{},
&PruneProviderTransformer{}, &PruneProviderTransformer{},
&DisableProviderTransformer{}, &DisableProviderTransformer{},
// Provisioner-related transformations // Provisioner-related transformations
&MissingProvisionerTransformer{Provisioners: b.Provisioners}, &MissingProvisionerTransformer{Provisioners: b.Provisioners},
&ProvisionerTransformer{}, &ProvisionerTransformer{},
&CloseProvisionerTransformer{},
&PruneProvisionerTransformer{}, &PruneProvisionerTransformer{},
// Run our vertex-level transforms // Run our vertex-level transforms

View File

@ -196,6 +196,8 @@ aws_instance.db
aws_instance.web aws_instance.web
aws_instance.db aws_instance.db
provider.aws provider.aws
provider.aws (close)
aws_instance.web
` `
const testBuiltinGraphBuilderVerboseStr = ` const testBuiltinGraphBuilderVerboseStr = `
@ -213,8 +215,28 @@ aws_instance.web (destroy tainted)
aws_instance.web (destroy) aws_instance.web (destroy)
provider.aws provider.aws
provider.aws provider.aws
provider.aws (close)
aws_instance.web
` `
const testBuiltinGraphBuilderMultiLevelStr = `
module.foo.module.bar.output.value
module.foo.module.bar.var.bar
module.foo.module.bar.plan-destroy
module.foo.module.bar.var.bar
module.foo.var.foo
module.foo.plan-destroy
module.foo.var.foo
root
module.foo.module.bar.output.value
module.foo.module.bar.plan-destroy
module.foo.plan-destroy
`
/*
TODO: Commented out this const as it's likely this needs to
be updated when the TestBuiltinGraphBuilder_modules test is
enabled again.
const testBuiltinGraphBuilderModuleStr = ` const testBuiltinGraphBuilderModuleStr = `
aws_instance.web aws_instance.web
aws_instance.web (destroy) aws_instance.web (destroy)
@ -231,17 +253,4 @@ module.consul (expanded)
provider.aws provider.aws
provider.aws provider.aws
` `
*/
const testBuiltinGraphBuilderMultiLevelStr = `
module.foo.module.bar.output.value
module.foo.module.bar.var.bar
module.foo.module.bar.plan-destroy
module.foo.module.bar.var.bar
module.foo.var.foo
module.foo.plan-destroy
module.foo.var.foo
root
module.foo.module.bar.output.value
module.foo.module.bar.plan-destroy
module.foo.plan-destroy
`

View File

@ -71,6 +71,12 @@ type ResourceProvider interface {
Refresh(*InstanceInfo, *InstanceState) (*InstanceState, error) Refresh(*InstanceInfo, *InstanceState) (*InstanceState, error)
} }
// ResourceProviderCloser is an interface that providers that can close
// connections that aren't needed anymore must implement.
type ResourceProviderCloser interface {
Close() error
}
// ResourceType is a type of resource that a resource provider can manage. // ResourceType is a type of resource that a resource provider can manage.
type ResourceType struct { type ResourceType struct {
Name string Name string

View File

@ -23,6 +23,12 @@ type ResourceProvisioner interface {
Apply(UIOutput, *InstanceState, *ResourceConfig) error Apply(UIOutput, *InstanceState, *ResourceConfig) error
} }
// ResourceProvisionerCloser is an interface that provisioners that can close
// connections that aren't needed anymore must implement.
type ResourceProvisionerCloser interface {
Close() error
}
// ResourceProvisionerFactory is a function type that creates a new instance // ResourceProvisionerFactory is a function type that creates a new instance
// of a resource provisioner. // of a resource provisioner.
type ResourceProvisionerFactory func() (ResourceProvisioner, error) type ResourceProvisionerFactory func() (ResourceProvisioner, error)

View File

@ -17,6 +17,13 @@ type GraphNodeProvider interface {
ProviderConfig() *config.RawConfig ProviderConfig() *config.RawConfig
} }
// GraphNodeCloseProvider is an interface that nodes that can be a close
// provider must implement. The CloseProviderName returned is the name of
// the provider they satisfy.
type GraphNodeCloseProvider interface {
CloseProviderName() string
}
// GraphNodeProviderConsumer is an interface that nodes that require // GraphNodeProviderConsumer is an interface that nodes that require
// a provider must implement. ProvidedBy must return the name of the provider // a provider must implement. ProvidedBy must return the name of the provider
// to use. // to use.
@ -98,6 +105,37 @@ func (t *ProviderTransformer) Transform(g *Graph) error {
return err return err
} }
// CloseProviderTransformer is a GraphTransformer that adds nodes to the
// graph that will close open provider connections that aren't needed anymore.
// A provider connection is not needed anymore once all depended resources
// in the graph are evaluated.
type CloseProviderTransformer struct{}
func (t *CloseProviderTransformer) Transform(g *Graph) error {
m := closeProviderVertexMap(g)
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeProviderConsumer); ok {
for _, p := range pv.ProvidedBy() {
source := m[p]
if source == nil {
// Create a new graphNodeCloseProvider and add it to the graph
source = &graphNodeCloseProvider{ProviderNameValue: p}
g.Add(source)
// Make sure we also add the new graphNodeCloseProvider to the map
// so we don't create and add any duplicate graphNodeCloseProviders.
m[p] = source
}
g.Connect(dag.BasicEdge(source, v))
}
}
}
return nil
}
// MissingProviderTransformer is a GraphTransformer that adds nodes // MissingProviderTransformer is a GraphTransformer that adds nodes
// for missing providers into the graph. Specifically, it creates provider // for missing providers into the graph. Specifically, it creates provider
// configuration nodes for all the providers that we support. These are // configuration nodes for all the providers that we support. These are
@ -143,6 +181,28 @@ func (t *PruneProviderTransformer) Transform(g *Graph) error {
return nil return nil
} }
func providerVertexMap(g *Graph) map[string]dag.Vertex {
m := make(map[string]dag.Vertex)
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeProvider); ok {
m[pv.ProviderName()] = v
}
}
return m
}
func closeProviderVertexMap(g *Graph) map[string]dag.Vertex {
m := make(map[string]dag.Vertex)
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeCloseProvider); ok {
m[pv.CloseProviderName()] = v
}
}
return m
}
type graphNodeDisabledProvider struct { type graphNodeDisabledProvider struct {
GraphNodeProvider GraphNodeProvider
} }
@ -258,6 +318,94 @@ func (n *graphNodeDisabledProviderFlat) DependentOn() []string {
return result return result
} }
type graphNodeCloseProvider struct {
ProviderNameValue string
}
func (n *graphNodeCloseProvider) Name() string {
return fmt.Sprintf("provider.%s (close)", n.ProviderNameValue)
}
// GraphNodeEvalable impl.
func (n *graphNodeCloseProvider) EvalTree() EvalNode {
return CloseProviderEvalTree(n.ProviderNameValue)
}
// GraphNodeDependable impl.
func (n *graphNodeCloseProvider) DependableName() []string {
return []string{n.Name()}
}
func (n *graphNodeCloseProvider) CloseProviderName() string {
return n.ProviderNameValue
}
// GraphNodeDotter impl.
func (n *graphNodeCloseProvider) DotNode(name string, opts *GraphDotOpts) *dot.Node {
return dot.NewNode(name, map[string]string{
"label": n.Name(),
"shape": "diamond",
})
}
// GraphNodeDotterOrigin impl.
func (n *graphNodeCloseProvider) DotOrigin() bool {
return true
}
// GraphNodeFlattenable impl.
func (n *graphNodeCloseProvider) Flatten(p []string) (dag.Vertex, error) {
return &graphNodeCloseProviderFlat{
graphNodeCloseProvider: n,
PathValue: p,
}, nil
}
// Same as graphNodeCloseProvider, but for flattening
type graphNodeCloseProviderFlat struct {
*graphNodeCloseProvider
PathValue []string
}
func (n *graphNodeCloseProviderFlat) Name() string {
return fmt.Sprintf(
"%s.%s", modulePrefixStr(n.PathValue), n.graphNodeCloseProvider.Name())
}
func (n *graphNodeCloseProviderFlat) Path() []string {
return n.PathValue
}
func (n *graphNodeCloseProviderFlat) ProviderName() string {
return fmt.Sprintf(
"%s.%s", modulePrefixStr(n.PathValue),
n.graphNodeCloseProvider.CloseProviderName())
}
// GraphNodeDependable impl.
func (n *graphNodeCloseProviderFlat) DependableName() []string {
return []string{n.Name()}
}
func (n *graphNodeCloseProviderFlat) DependentOn() []string {
var result []string
// If we're in a module, then depend on our parent's provider
if len(n.PathValue) > 1 {
prefix := modulePrefixStr(n.PathValue[:len(n.PathValue)-1])
if prefix != "" {
prefix += "."
}
result = append(result, fmt.Sprintf(
"%s%s",
prefix, n.graphNodeCloseProvider.Name()))
}
return result
}
type graphNodeMissingProvider struct { type graphNodeMissingProvider struct {
ProviderNameValue string ProviderNameValue string
} }
@ -305,17 +453,6 @@ func (n *graphNodeMissingProvider) Flatten(p []string) (dag.Vertex, error) {
}, nil }, nil
} }
func providerVertexMap(g *Graph) map[string]dag.Vertex {
m := make(map[string]dag.Vertex)
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeProvider); ok {
m[pv.ProviderName()] = v
}
}
return m
}
// Same as graphNodeMissingProvider, but for flattening // Same as graphNodeMissingProvider, but for flattening
type graphNodeMissingProviderFlat struct { type graphNodeMissingProviderFlat struct {
*graphNodeMissingProvider *graphNodeMissingProvider

View File

@ -30,6 +30,38 @@ func TestProviderTransformer(t *testing.T) {
} }
} }
func TestCloseProviderTransformer(t *testing.T) {
mod := testModule(t, "transform-provider-basic")
g := Graph{Path: RootModulePath}
{
tf := &ConfigTransformer{Module: mod}
if err := tf.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
{
transform := &ProviderTransformer{}
if err := transform.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
{
transform := &CloseProviderTransformer{}
if err := transform.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
actual := strings.TrimSpace(g.String())
expected := strings.TrimSpace(testTransformCloseProviderBasicStr)
if actual != expected {
t.Fatalf("bad:\n\n%s", actual)
}
}
func TestMissingProviderTransformer(t *testing.T) { func TestMissingProviderTransformer(t *testing.T) {
mod := testModule(t, "transform-provider-basic") mod := testModule(t, "transform-provider-basic")
@ -41,9 +73,18 @@ func TestMissingProviderTransformer(t *testing.T) {
} }
} }
transform := &MissingProviderTransformer{Providers: []string{"foo"}} {
if err := transform.Transform(&g); err != nil { transform := &MissingProviderTransformer{Providers: []string{"foo"}}
t.Fatalf("err: %s", err) if err := transform.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
{
transform := &CloseProviderTransformer{}
if err := transform.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
} }
actual := strings.TrimSpace(g.String()) actual := strings.TrimSpace(g.String())
@ -78,6 +119,13 @@ func TestPruneProviderTransformer(t *testing.T) {
} }
} }
{
transform := &CloseProviderTransformer{}
if err := transform.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
{ {
transform := &PruneProviderTransformer{} transform := &PruneProviderTransformer{}
if err := transform.Transform(&g); err != nil { if err := transform.Transform(&g); err != nil {
@ -117,6 +165,13 @@ func TestDisableProviderTransformer(t *testing.T) {
} }
} }
{
transform := &CloseProviderTransformer{}
if err := transform.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
{ {
transform := &PruneProviderTransformer{} transform := &PruneProviderTransformer{}
if err := transform.Transform(&g); err != nil { if err := transform.Transform(&g); err != nil {
@ -163,6 +218,13 @@ func TestDisableProviderTransformer_keep(t *testing.T) {
} }
} }
{
transform := &CloseProviderTransformer{}
if err := transform.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
{ {
transform := &PruneProviderTransformer{} transform := &PruneProviderTransformer{}
if err := transform.Transform(&g); err != nil { if err := transform.Transform(&g); err != nil {
@ -203,9 +265,19 @@ aws_instance.web
provider.aws provider.aws
` `
const testTransformCloseProviderBasicStr = `
aws_instance.web
provider.aws
provider.aws
provider.aws (close)
aws_instance.web
`
const testTransformMissingProviderBasicStr = ` const testTransformMissingProviderBasicStr = `
aws_instance.web aws_instance.web
provider.aws provider.aws
provider.aws (close)
aws_instance.web
provider.foo provider.foo
` `
@ -213,12 +285,16 @@ const testTransformPruneProviderBasicStr = `
foo_instance.web foo_instance.web
provider.foo provider.foo
provider.foo provider.foo
provider.foo (close)
foo_instance.web
` `
const testTransformDisableProviderBasicStr = ` const testTransformDisableProviderBasicStr = `
module.child module.child
provider.aws (disabled) provider.aws (disabled)
var.foo var.foo
provider.aws (close)
module.child
provider.aws (disabled) provider.aws (disabled)
var.foo var.foo
` `
@ -230,5 +306,8 @@ module.child
provider.aws provider.aws
var.foo var.foo
provider.aws provider.aws
provider.aws (close)
aws_instance.foo
module.child
var.foo var.foo
` `

View File

@ -14,6 +14,13 @@ type GraphNodeProvisioner interface {
ProvisionerName() string ProvisionerName() string
} }
// GraphNodeCloseProvisioner is an interface that nodes that can be a close
// provisioner must implement. The CloseProvisionerName returned is the name
// of the provisioner they satisfy.
type GraphNodeCloseProvisioner interface {
CloseProvisionerName() string
}
// GraphNodeProvisionerConsumer is an interface that nodes that require // GraphNodeProvisionerConsumer is an interface that nodes that require
// a provisioner must implement. ProvisionedBy must return the name of the // a provisioner must implement. ProvisionedBy must return the name of the
// provisioner to use. // provisioner to use.
@ -49,6 +56,37 @@ func (t *ProvisionerTransformer) Transform(g *Graph) error {
return err return err
} }
// CloseProvisionerTransformer is a GraphTransformer that adds nodes to the
// graph that will close open provisioner connections that aren't needed
// anymore. A provisioner connection is not needed anymore once all depended
// resources in the graph are evaluated.
type CloseProvisionerTransformer struct{}
func (t *CloseProvisionerTransformer) Transform(g *Graph) error {
m := closeProvisionerVertexMap(g)
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeProvisionerConsumer); ok {
for _, p := range pv.ProvisionedBy() {
source := m[p]
if source == nil {
// Create a new graphNodeCloseProvisioner and add it to the graph
source = &graphNodeCloseProvisioner{ProvisionerNameValue: p}
g.Add(source)
// Make sure we also add the new graphNodeCloseProvisioner to the map
// so we don't create and add any duplicate graphNodeCloseProvisioners.
m[p] = source
}
g.Connect(dag.BasicEdge(source, v))
}
}
}
return nil
}
// MissingProvisionerTransformer is a GraphTransformer that adds nodes // MissingProvisionerTransformer is a GraphTransformer that adds nodes
// for missing provisioners into the graph. Specifically, it creates provisioner // for missing provisioners into the graph. Specifically, it creates provisioner
// configuration nodes for all the provisioners that we support. These are // configuration nodes for all the provisioners that we support. These are
@ -94,6 +132,75 @@ func (t *PruneProvisionerTransformer) Transform(g *Graph) error {
return nil return nil
} }
func provisionerVertexMap(g *Graph) map[string]dag.Vertex {
m := make(map[string]dag.Vertex)
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeProvisioner); ok {
m[pv.ProvisionerName()] = v
}
}
return m
}
func closeProvisionerVertexMap(g *Graph) map[string]dag.Vertex {
m := make(map[string]dag.Vertex)
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeCloseProvisioner); ok {
m[pv.CloseProvisionerName()] = v
}
}
return m
}
type graphNodeCloseProvisioner struct {
ProvisionerNameValue string
}
func (n *graphNodeCloseProvisioner) Name() string {
return fmt.Sprintf("provisioner.%s (close)", n.ProvisionerNameValue)
}
// GraphNodeEvalable impl.
func (n *graphNodeCloseProvisioner) EvalTree() EvalNode {
return &EvalCloseProvisioner{Name: n.ProvisionerNameValue}
}
func (n *graphNodeCloseProvisioner) CloseProvisionerName() string {
return n.ProvisionerNameValue
}
// GraphNodeFlattenable impl.
func (n *graphNodeCloseProvisioner) Flatten(p []string) (dag.Vertex, error) {
return &graphNodeCloseProvisionerFlat{
graphNodeCloseProvisioner: n,
PathValue: p,
}, nil
}
// Same as graphNodeCloseProvisioner, but for flattening
type graphNodeCloseProvisionerFlat struct {
*graphNodeCloseProvisioner
PathValue []string
}
func (n *graphNodeCloseProvisionerFlat) Name() string {
return fmt.Sprintf(
"%s.%s", modulePrefixStr(n.PathValue), n.graphNodeCloseProvisioner.Name())
}
func (n *graphNodeCloseProvisionerFlat) Path() []string {
return n.PathValue
}
func (n *graphNodeCloseProvisionerFlat) ProvisionerName() string {
return fmt.Sprintf(
"%s.%s", modulePrefixStr(n.PathValue),
n.graphNodeCloseProvisioner.CloseProvisionerName())
}
type graphNodeMissingProvisioner struct { type graphNodeMissingProvisioner struct {
ProvisionerNameValue string ProvisionerNameValue string
} }
@ -119,17 +226,6 @@ func (n *graphNodeMissingProvisioner) Flatten(p []string) (dag.Vertex, error) {
}, nil }, nil
} }
func provisionerVertexMap(g *Graph) map[string]dag.Vertex {
m := make(map[string]dag.Vertex)
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeProvisioner); ok {
m[pv.ProvisionerName()] = v
}
}
return m
}
// Same as graphNodeMissingProvisioner, but for flattening // Same as graphNodeMissingProvisioner, but for flattening
type graphNodeMissingProvisionerFlat struct { type graphNodeMissingProvisionerFlat struct {
*graphNodeMissingProvisioner *graphNodeMissingProvisioner

View File

@ -18,9 +18,18 @@ func TestMissingProvisionerTransformer(t *testing.T) {
} }
} }
transform := &MissingProvisionerTransformer{Provisioners: []string{"foo"}} {
if err := transform.Transform(&g); err != nil { transform := &MissingProvisionerTransformer{Provisioners: []string{"foo"}}
t.Fatalf("err: %s", err) if err := transform.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
{
transform := &CloseProvisionerTransformer{}
if err := transform.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
} }
actual := strings.TrimSpace(g.String()) actual := strings.TrimSpace(g.String())
@ -56,6 +65,13 @@ func TestPruneProvisionerTransformer(t *testing.T) {
} }
} }
{
transform := &CloseProvisionerTransformer{}
if err := transform.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
{ {
transform := &PruneProvisionerTransformer{} transform := &PruneProvisionerTransformer{}
if err := transform.Transform(&g); err != nil { if err := transform.Transform(&g); err != nil {
@ -86,10 +102,14 @@ func TestGraphNodeMissingProvisioner_ProvisionerName(t *testing.T) {
const testTransformMissingProvisionerBasicStr = ` const testTransformMissingProvisionerBasicStr = `
aws_instance.web aws_instance.web
provisioner.foo provisioner.foo
provisioner.shell (close)
aws_instance.web
` `
const testTransformPruneProvisionerBasicStr = ` const testTransformPruneProvisionerBasicStr = `
aws_instance.web aws_instance.web
provisioner.foo provisioner.foo
provisioner.foo provisioner.foo
provisioner.foo (close)
aws_instance.web
` `