Merge pull request #13 from hashicorp/f-count

Count Meta-Parameter
This commit is contained in:
Mitchell Hashimoto 2014-07-03 21:59:14 -07:00
commit 338bafc892
17 changed files with 533 additions and 33 deletions

View File

@ -31,6 +31,7 @@ type ProviderConfig struct {
type Resource struct {
Name string
Type string
Count int
RawConfig *RawConfig
}

View File

@ -253,6 +253,9 @@ func loadResourcesLibucl(o *libucl.Object) ([]*Resource, error) {
err)
}
// Remove the "count" from the config, since we treat that special
delete(config, "count")
rawConfig, err := NewRawConfig(config)
if err != nil {
return nil, fmt.Errorf(
@ -262,9 +265,24 @@ func loadResourcesLibucl(o *libucl.Object) ([]*Resource, error) {
err)
}
// If we have a count, then figure it out
var count int = 1
if o := r.Get("count"); o != nil {
err = o.Decode(&count)
o.Close()
if err != nil {
return nil, fmt.Errorf(
"Error parsing count for %s[%s]: %s",
t.Key(),
r.Key(),
err)
}
}
result = append(result, &Resource{
Name: r.Key(),
Type: t.Key(),
Count: count,
RawConfig: rawConfig,
})
}

View File

@ -145,9 +145,10 @@ func resourcesStr(rs []*Resource) string {
result := ""
for _, r := range rs {
result += fmt.Sprintf(
"%s[%s]\n",
"%s[%s] (x%d)\n",
r.Type,
r.Name)
r.Name,
r.Count)
ks := make([]string, 0, len(r.RawConfig.Raw))
for k, _ := range r.RawConfig.Raw {
@ -229,14 +230,18 @@ do
`
const basicResourcesStr = `
aws_security_group[firewall]
aws_instance[web]
aws_security_group[firewall] (x5)
aws_instance[web] (x1)
ami
network_interface
security_groups
vars
resource: aws_security_group.firewall.foo
user: var.foo
aws_instance[db] (x1)
security_groups
vars
resource: aws_security_group.firewall.*.id
`
const basicVariablesStr = `
@ -251,8 +256,8 @@ aws
`
const importResourcesStr = `
aws_security_group[db]
aws_security_group[web]
aws_security_group[db] (x1)
aws_security_group[web] (x1)
`
const importVariablesStr = `

View File

@ -13,6 +13,7 @@ provider "do" {
}
resource "aws_security_group" "firewall" {
count = 5
}
resource aws_instance "web" {
@ -27,3 +28,7 @@ resource aws_instance "web" {
description = "Main network interface"
}
}
resource "aws_instance" "db" {
security_groups = "${aws_security_group.firewall.*.id}"
}

View File

@ -13,7 +13,7 @@ import (
var varRegexp *regexp.Regexp
func init() {
varRegexp = regexp.MustCompile(`(?i)(\$+)\{([-.a-z0-9_]+)\}`)
varRegexp = regexp.MustCompile(`(?i)(\$+)\{([*-.a-z0-9_]+)\}`)
}
// ReplaceVariables takes a configuration and a mapping of variables

View File

@ -87,6 +87,22 @@ func TestVariableDetectWalker_resource(t *testing.T) {
}
}
func TestVariableDetectWalker_resourceMulti(t *testing.T) {
w := new(variableDetectWalker)
str := `foo ${ec2.foo.*.bar}`
if err := w.Primitive(reflect.ValueOf(str)); err != nil {
t.Fatalf("err: %s", err)
}
if len(w.Variables) != 1 {
t.Fatalf("bad: %#v", w.Variables)
}
if w.Variables["ec2.foo.*.bar"].(*ResourceVariable).FullKey() != "ec2.foo.*.bar" {
t.Fatalf("bad: %#v", w.Variables)
}
}
func TestVariableDetectWalker_bad(t *testing.T) {
w := new(variableDetectWalker)

View File

@ -9,6 +9,7 @@ import (
"bytes"
"fmt"
"sort"
"strings"
"sync"
"github.com/hashicorp/terraform/digraph"
@ -42,7 +43,34 @@ type ValidateError struct {
}
func (v *ValidateError) Error() string {
return "The depedency graph is not valid"
var msgs []string
if v.MissingRoot {
msgs = append(msgs, "The graph has no single root")
}
for _, n := range v.Unreachable {
msgs = append(msgs, fmt.Sprintf(
"Unreachable node: %s", n.Name))
}
for _, c := range v.Cycles {
cycleNodes := make([]string, len(c))
for i, n := range c {
cycleNodes[i] = n.Name
}
msgs = append(msgs, fmt.Sprintf(
"Cycle: %s", strings.Join(cycleNodes, " -> ")))
}
for i, m := range msgs {
msgs[i] = fmt.Sprintf("* %s", m)
}
return fmt.Sprintf(
"The dependency graph is not valid:\n\n%s",
strings.Join(msgs, "\n"))
}
// ConstraintError is used to return detailed violation

View File

@ -3,6 +3,7 @@ package terraform
import (
"fmt"
"log"
"strings"
"sync"
"sync/atomic"
@ -509,6 +510,9 @@ func (c *Context) genericWalkFn(
vars[fmt.Sprintf("var.%s", k)] = v
}
// This will keep track of the counts of multi-count resources
counts := make(map[string]int)
// This will keep track of whether we're stopped or not
var stop uint32 = 0
@ -523,8 +527,20 @@ func (c *Context) genericWalkFn(
return nil
}
// Calculate any aggregate interpolated variables if we have to.
// Aggregate variables (such as "test_instance.foo.*.id") are not
// pre-computed since the fanout would be expensive. We calculate
// them on-demand here.
computeAggregateVars(&l, n, counts, vars)
switch m := n.Meta.(type) {
case *GraphNodeResource:
case *GraphNodeResourceMeta:
// Record the count and then just ignore
l.Lock()
counts[m.ID] = m.Count
l.Unlock()
return nil
case *GraphNodeResourceProvider:
var rc *ResourceConfig
if m.Config != nil {
@ -543,6 +559,8 @@ func (c *Context) genericWalkFn(
}
return nil
default:
panic(fmt.Sprintf("unknown graph node: %#v", n.Meta))
}
rn := n.Meta.(*GraphNodeResource)
@ -603,3 +621,68 @@ func (c *Context) genericWalkFn(
return nil
}
}
func computeAggregateVars(
l *sync.RWMutex,
n *depgraph.Noun,
cs map[string]int,
vs map[string]string) {
var ivars map[string]config.InterpolatedVariable
switch m := n.Meta.(type) {
case *GraphNodeResource:
if m.Config != nil {
ivars = m.Config.RawConfig.Variables
}
case *GraphNodeResourceProvider:
if m.Config != nil {
ivars = m.Config.RawConfig.Variables
}
}
if len(ivars) == 0 {
return
}
for _, v := range ivars {
rv, ok := v.(*config.ResourceVariable)
if !ok {
continue
}
idx := strings.Index(rv.Field, ".")
if idx == -1 {
// It isn't an aggregated var
continue
}
if rv.Field[:idx] != "*" {
// It isn't an aggregated var
continue
}
field := rv.Field[idx+1:]
// Get the meta node so that we can determine the count
key := fmt.Sprintf("%s.%s", rv.Type, rv.Name)
l.RLock()
count, ok := cs[key]
l.RUnlock()
if !ok {
// This should never happen due to semantic checks
panic(fmt.Sprintf(
"non-existent resource variable access: %s\n\n%#v", key, rv))
}
var values []string
for i := 0; i < count; i++ {
key := fmt.Sprintf(
"%s.%s.%d.%s",
rv.Type,
rv.Name,
i,
field)
if v, ok := vs[key]; ok {
values = append(values, v)
}
}
vs[rv.FullKey()] = strings.Join(values, ",")
}
}

View File

@ -641,6 +641,113 @@ func TestContextPlan_computed(t *testing.T) {
}
}
func TestContextPlan_count(t *testing.T) {
c := testConfig(t, "plan-count")
p := testProvider("aws")
p.DiffFn = testDiffFn
ctx := testContext(t, &ContextOpts{
Config: c,
Providers: map[string]ResourceProviderFactory{
"aws": testProviderFuncFixed(p),
},
})
plan, err := ctx.Plan(nil)
if err != nil {
t.Fatalf("err: %s", err)
}
if len(plan.Diff.Resources) < 6 {
t.Fatalf("bad: %#v", plan.Diff.Resources)
}
actual := strings.TrimSpace(plan.String())
expected := strings.TrimSpace(testTerraformPlanCountStr)
if actual != expected {
t.Fatalf("bad:\n%s", actual)
}
}
func TestContextPlan_countDecreaseToOne(t *testing.T) {
c := testConfig(t, "plan-count-dec")
p := testProvider("aws")
p.DiffFn = testDiffFn
s := &State{
Resources: map[string]*ResourceState{
"aws_instance.foo.0": &ResourceState{
ID: "bar",
Type: "aws_instance",
Attributes: map[string]string{
"foo": "foo",
"type": "aws_instance",
},
},
"aws_instance.foo.1": &ResourceState{
ID: "bar",
Type: "aws_instance",
},
"aws_instance.foo.2": &ResourceState{
ID: "bar",
Type: "aws_instance",
},
},
}
ctx := testContext(t, &ContextOpts{
Config: c,
Providers: map[string]ResourceProviderFactory{
"aws": testProviderFuncFixed(p),
},
State: s,
})
plan, err := ctx.Plan(nil)
if err != nil {
t.Fatalf("err: %s", err)
}
actual := strings.TrimSpace(plan.String())
expected := strings.TrimSpace(testTerraformPlanCountDecreaseStr)
if actual != expected {
t.Fatalf("bad:\n%s", actual)
}
}
func TestContextPlan_countIncreaseFromOne(t *testing.T) {
c := testConfig(t, "plan-count-inc")
p := testProvider("aws")
p.DiffFn = testDiffFn
s := &State{
Resources: map[string]*ResourceState{
"aws_instance.foo": &ResourceState{
ID: "bar",
Type: "aws_instance",
Attributes: map[string]string{
"foo": "foo",
"type": "aws_instance",
},
},
},
}
ctx := testContext(t, &ContextOpts{
Config: c,
Providers: map[string]ResourceProviderFactory{
"aws": testProviderFuncFixed(p),
},
State: s,
})
plan, err := ctx.Plan(nil)
if err != nil {
t.Fatalf("err: %s", err)
}
actual := strings.TrimSpace(plan.String())
expected := strings.TrimSpace(testTerraformPlanCountIncreaseStr)
if actual != expected {
t.Fatalf("bad:\n%s", actual)
}
}
func TestContextPlan_destroy(t *testing.T) {
c := testConfig(t, "plan-destroy")
p := testProvider("aws")
@ -917,10 +1024,6 @@ func testDiffFn(
c *ResourceConfig) (*ResourceDiff, error) {
var diff ResourceDiff
diff.Attributes = make(map[string]*ResourceAttrDiff)
diff.Attributes["type"] = &ResourceAttrDiff{
Old: "",
New: s.Type,
}
for k, v := range c.Raw {
if _, ok := v.(string); !ok {
@ -982,6 +1085,27 @@ func testDiffFn(
}
}
for k, v := range diff.Attributes {
if v.NewComputed {
continue
}
old, ok := s.Attributes[k]
if !ok {
continue
}
if old == v.New {
delete(diff.Attributes, k)
}
}
if !diff.Empty() {
diff.Attributes["type"] = &ResourceAttrDiff{
Old: "",
New: s.Type,
}
}
return &diff, nil
}

View File

@ -45,8 +45,12 @@ type GraphOpts struct {
// graph. This node is just a placemarker and has no associated functionality.
const GraphRootNode = "root"
// GraphNodeResource is a node type in the graph that represents a resource.
// GraphNodeResource is a node type in the graph that represents a resource
// that will be created or managed. Unlike the GraphNodeResourceMeta node,
// this represents a _single_, _resource_ to be managed, not a set of resources
// or a component of a resource.
type GraphNodeResource struct {
Index int
Type string
Config *config.Resource
Orphan bool
@ -54,6 +58,16 @@ type GraphNodeResource struct {
ResourceProviderID string
}
// GraphNodeResourceMeta is a node type in the graph that represents the
// metadata for a resource. There will be one meta node for every resource
// in the configuration.
type GraphNodeResourceMeta struct {
ID string
Name string
Type string
Count int
}
// GraphNodeResourceProvider is a node type in the graph that represents
// the configuration for a resource provider.
type GraphNodeResourceProvider struct {
@ -142,28 +156,85 @@ func graphAddConfigResources(
// This tracks all the resource nouns
nouns := make(map[string]*depgraph.Noun)
for _, r := range c.Resources {
var state *ResourceState
if s != nil {
state = s.Resources[r.Id()]
}
if state == nil {
state = &ResourceState{
Type: r.Type,
resourceNouns := make([]*depgraph.Noun, r.Count)
for i := 0; i < r.Count; i++ {
name := r.Id()
index := -1
// If we have a count that is more than one, then make sure
// we suffix with the number of the resource that this is.
if r.Count > 1 {
name = fmt.Sprintf("%s.%d", name, i)
index = i
}
var state *ResourceState
if s != nil {
state = s.Resources[name]
if state == nil {
if r.Count == 1 {
// If the count is one, check the state for ".0"
// appended, which might exist if we go from
// count > 1 to count == 1.
state = s.Resources[r.Id()+".0"]
} else if i == 0 {
// If count is greater than one, check for state
// with just the ID, which might exist if we go
// from count == 1 to count > 1
state = s.Resources[r.Id()]
}
}
}
if state == nil {
state = &ResourceState{
Type: r.Type,
}
}
resourceNouns[i] = &depgraph.Noun{
Name: name,
Meta: &GraphNodeResource{
Index: index,
Type: r.Type,
Config: r,
Resource: &Resource{
Id: name,
State: state,
},
},
}
}
noun := &depgraph.Noun{
Name: r.Id(),
Meta: &GraphNodeResource{
Type: r.Type,
Config: r,
Resource: &Resource{
Id: r.Id(),
State: state,
// If we have more than one, then create a meta node to track
// the resources.
if r.Count > 1 {
metaNoun := &depgraph.Noun{
Name: r.Id(),
Meta: &GraphNodeResourceMeta{
ID: r.Id(),
Name: r.Name,
Type: r.Type,
Count: r.Count,
},
},
}
// Create the dependencies on this noun
for _, n := range resourceNouns {
metaNoun.Deps = append(metaNoun.Deps, &depgraph.Dependency{
Name: n.Name,
Source: metaNoun,
Target: n,
})
}
// Assign it to the map so that we have it
nouns[metaNoun.Name] = metaNoun
}
for _, n := range resourceNouns {
nouns[n.Name] = n
}
nouns[noun.Name] = noun
}
// Build the list of nouns that we iterate over
@ -339,6 +410,7 @@ func graphAddOrphans(g *depgraph.Graph, c *config.Config, s *State) {
noun := &depgraph.Noun{
Name: k,
Meta: &GraphNodeResource{
Index: -1,
Type: rs.Type,
Orphan: true,
Resource: &Resource{
@ -357,7 +429,10 @@ func graphAddProviderConfigs(g *depgraph.Graph, c *config.Config) {
nounsList := make([]*depgraph.Noun, 0, 2)
pcNouns := make(map[string]*depgraph.Noun)
for _, noun := range g.Nouns {
resourceNode := noun.Meta.(*GraphNodeResource)
resourceNode, ok := noun.Meta.(*GraphNodeResource)
if !ok {
continue
}
// Look up the provider config for this resource
pcName := config.ProviderConfigName(resourceNode.Type, c.ProviderConfigs)
@ -401,8 +476,17 @@ func graphAddProviderConfigs(g *depgraph.Graph, c *config.Config) {
func graphAddRoot(g *depgraph.Graph) {
root := &depgraph.Noun{Name: GraphRootNode}
for _, n := range g.Nouns {
// The root only needs to depend on all the resources
if _, ok := n.Meta.(*GraphNodeResource); !ok {
switch m := n.Meta.(type) {
case *GraphNodeResource:
// If the resource is part of a group, we don't need to make a dep
if m.Index != -1 {
continue
}
case *GraphNodeResourceMeta:
// Always in the graph
case *GraphNodeResourceProvider:
// ResourceProviders don't need to be in the root deps because
// they're always pointed to by some resource.
continue
}

View File

@ -27,6 +27,21 @@ func TestGraph_configRequired(t *testing.T) {
}
}
func TestGraph_count(t *testing.T) {
config := testConfig(t, "graph-count")
g, err := Graph(&GraphOpts{Config: config})
if err != nil {
t.Fatalf("err: %s", err)
}
actual := strings.TrimSpace(g.String())
expected := strings.TrimSpace(testTerraformGraphCountStr)
if actual != expected {
t.Fatalf("bad:\n\n%s", actual)
}
}
func TestGraph_cycle(t *testing.T) {
config := testConfig(t, "graph-cycle")
@ -226,6 +241,22 @@ root
root -> openstack_floating_ip.random
`
const testTerraformGraphCountStr = `
root: root
aws_instance.web
aws_instance.web -> aws_instance.web.0
aws_instance.web -> aws_instance.web.1
aws_instance.web -> aws_instance.web.2
aws_instance.web.0
aws_instance.web.1
aws_instance.web.2
aws_load_balancer.weblb
aws_load_balancer.weblb -> aws_instance.web
root
root -> aws_instance.web
root -> aws_load_balancer.weblb
`
const testTerraformGraphDiffStr = `
root: root
aws_instance.foo

View File

@ -38,6 +38,12 @@ func (s *State) Orphans(c *config.Config) []string {
for _, r := range c.Resources {
delete(keys, r.Id())
// If there is only one of this instance, then we alias that
// to the ".0" version as well so that it can count
if r.Count == 1 {
delete(keys, r.Id()+".0")
}
}
result := make([]string, 0, len(keys))

View File

@ -155,6 +155,75 @@ STATE:
<no state>
`
const testTerraformPlanCountStr = `
DIFF:
UPDATE: aws_instance.bar
foo: "" => "foo,foo,foo,foo,foo"
type: "" => "aws_instance"
UPDATE: aws_instance.foo.0
foo: "" => "foo"
type: "" => "aws_instance"
UPDATE: aws_instance.foo.1
foo: "" => "foo"
type: "" => "aws_instance"
UPDATE: aws_instance.foo.2
foo: "" => "foo"
type: "" => "aws_instance"
UPDATE: aws_instance.foo.3
foo: "" => "foo"
type: "" => "aws_instance"
UPDATE: aws_instance.foo.4
foo: "" => "foo"
type: "" => "aws_instance"
STATE:
<no state>
`
const testTerraformPlanCountDecreaseStr = `
DIFF:
UPDATE: aws_instance.bar
foo: "" => "bar"
type: "" => "aws_instance"
DESTROY: aws_instance.foo.1
DESTROY: aws_instance.foo.2
STATE:
aws_instance.foo.0:
ID = bar
foo = foo
type = aws_instance
aws_instance.foo.1:
ID = bar
aws_instance.foo.2:
ID = bar
`
const testTerraformPlanCountIncreaseStr = `
DIFF:
UPDATE: aws_instance.bar
foo: "" => "bar"
type: "" => "aws_instance"
UPDATE: aws_instance.foo.1
foo: "" => "foo"
type: "" => "aws_instance"
UPDATE: aws_instance.foo.2
foo: "" => "foo"
type: "" => "aws_instance"
STATE:
aws_instance.foo:
ID = bar
foo = foo
type = aws_instance
`
const testTerraformPlanDestroyStr = `
DIFF:

View File

@ -0,0 +1,7 @@
resource "aws_instance" "web" {
count = 3
}
resource "aws_load_balancer" "weblb" {
members = "${aws_instance.web.*.id}"
}

View File

@ -0,0 +1,7 @@
resource "aws_instance" "foo" {
foo = "foo"
}
resource "aws_instance" "bar" {
foo = "bar"
}

View File

@ -0,0 +1,8 @@
resource "aws_instance" "foo" {
foo = "foo"
count = 3
}
resource "aws_instance" "bar" {
foo = "bar"
}

View File

@ -0,0 +1,8 @@
resource "aws_instance" "foo" {
count = 5
foo = "foo"
}
resource "aws_instance" "bar" {
foo = "${aws_instance.foo.*.foo}"
}