Merge pull request #32 from hashicorp/f-parser

Parser

.gitignore

@@ -4,4 +4,6 @@ example.tf
 terraform.tfplan
 terraform.tfstate
 bin/
+config/y.go
+config/y.output
 vendor/

Makefile

@@ -16,27 +16,31 @@ export CGO_CFLAGS CGO_LDFLAGS PATH
 
 default: test
 
-dev: libucl
+dev: config/y.go libucl
 	@sh -c "$(CURDIR)/scripts/build.sh"
 
 libucl: vendor/libucl/$(LIBUCL_NAME)
 
-test: libucl
+test: config/y.go libucl
 	TF_ACC= go test $(TEST) $(TESTARGS) -timeout=10s
 
-testacc: libucl
+testacc: config/y.go libucl
 	@if [ "$(TEST)" = "./..." ]; then \
 		echo "ERROR: Set TEST to a specific package"; \
 		exit 1; \
 	fi
 	TF_ACC=1 go test $(TEST) -v $(TESTARGS)
 
-testrace: libucl
+testrace: config/y.go libucl
 	TF_ACC= go test -race $(TEST) $(TESTARGS)
 
 updatedeps:
 	go get -u -v ./...
 
+config/y.go: config/expr.y
+	cd config/ && \
+		go tool yacc -p "expr" expr.y
+
 vendor/libucl/libucl.a: vendor/libucl
 	cd vendor/libucl && \
 		cmake cmake/ && \

config/expr.y

@@ -0,0 +1,91 @@
+// This is the yacc input for creating the parser for interpolation
+// expressions in Go.
+
+// To build it:
+//
+// go tool yacc -p "expr" expr.y (produces y.go)
+//
+
+%{
+package config
+
+import (
+	"fmt"
+)
+
+%}
+
+%union {
+	expr Interpolation
+    str string
+	variable InterpolatedVariable
+	args []Interpolation
+}
+
+%type	<args> args
+%type   <expr> expr
+%type   <str> string
+%type   <variable> variable
+
+%token  <str> STRING IDENTIFIER
+%token	<str> COMMA LEFTPAREN RIGHTPAREN
+
+%%
+
+top:
+	expr
+	{
+		exprResult = $1
+	}
+
+expr:
+	string
+	{
+		$$ = &LiteralInterpolation{Literal: $1}
+	}
+|	variable
+	{
+		$$ = &VariableInterpolation{Variable: $1}
+	}
+|	IDENTIFIER LEFTPAREN args RIGHTPAREN
+	{
+		f, ok := Funcs[$1]
+		if !ok {
+			exprErrors = append(exprErrors, fmt.Errorf(
+				"Unknown function: %s", $1))
+		}
+
+		$$ = &FunctionInterpolation{Func: f, Args: $3}
+	}
+
+args:
+	{
+		$$ = nil
+	}
+|	expr COMMA expr
+	{
+		$$ = append($$, $1, $3)
+	}
+|	expr
+	{
+		$$ = append($$, $1)
+	}
+
+string:
+	STRING
+	{
+		$$ = $1
+	}
+
+variable:
+	IDENTIFIER
+	{
+		var err error
+		$$, err = NewInterpolatedVariable($1)
+		if err != nil {
+			exprErrors = append(exprErrors, fmt.Errorf(
+				"Error parsing variable '%s': %s", $1, err))
+		}
+	}
+
+%%

config/expr_lex.go

@@ -0,0 +1,131 @@
+package config
+
+import (
+	"bytes"
+	"log"
+	"unicode"
+	"unicode/utf8"
+)
+
+// The parser expects the lexer to return 0 on EOF.
+const lexEOF = 0
+
+// The parser uses the type <prefix>Lex as a lexer.  It must provide
+// the methods Lex(*<prefix>SymType) int and Error(string).
+type exprLex struct {
+	input string
+	pos   int
+	width int
+}
+
+// The parser calls this method to get each new token.
+func (x *exprLex) Lex(yylval *exprSymType) int {
+	for {
+		c := x.next()
+		if c == lexEOF {
+			return lexEOF
+		}
+
+		// Ignore all whitespace
+		if unicode.IsSpace(c) {
+			continue
+		}
+
+		switch c {
+		case '"':
+			return x.lexString(yylval)
+		case ',':
+			return COMMA
+		case '(':
+			return LEFTPAREN
+		case ')':
+			return RIGHTPAREN
+		default:
+			x.backup()
+			return x.lexId(yylval)
+		}
+	}
+}
+
+func (x *exprLex) lexId(yylval *exprSymType) int {
+	var b bytes.Buffer
+	for {
+		c := x.next()
+		if c == lexEOF {
+			break
+		}
+
+		// If this isn't a character we want in an ID, return out.
+		// One day we should make this a regexp.
+		if c != '_' &&
+			c != '-' &&
+			c != '.' &&
+			c != '*' &&
+			!unicode.IsLetter(c) &&
+			!unicode.IsNumber(c) {
+			x.backup()
+			break
+		}
+
+		if _, err := b.WriteRune(c); err != nil {
+			log.Printf("ERR: %s", err)
+			return lexEOF
+		}
+	}
+
+	yylval.str = b.String()
+	return IDENTIFIER
+}
+
+func (x *exprLex) lexString(yylval *exprSymType) int {
+	var b bytes.Buffer
+	for {
+		c := x.next()
+		if c == lexEOF {
+			break
+		}
+
+		// String end
+		if c == '"' {
+			break
+		}
+
+		if _, err := b.WriteRune(c); err != nil {
+			log.Printf("ERR: %s", err)
+			return lexEOF
+		}
+	}
+
+	yylval.str = b.String()
+	return STRING
+}
+
+// Return the next rune for the lexer.
+func (x *exprLex) next() rune {
+	if int(x.pos) >= len(x.input) {
+		x.width = 0
+		return lexEOF
+	}
+
+	r, w := utf8.DecodeRuneInString(x.input[x.pos:])
+	x.width = w
+	x.pos += x.width
+	return r
+}
+
+// peek returns but does not consume the next rune in the input
+func (x *exprLex) peek() rune {
+	r := x.next()
+	x.backup()
+	return r
+}
+
+// backup steps back one rune. Can only be called once per next.
+func (x *exprLex) backup() {
+	x.pos -= x.width
+}
+
+// The parser calls this method on a parse error.
+func (x *exprLex) Error(s string) {
+	log.Printf("parse error: %s", s)
+}

config/expr_parse.go

@@ -0,0 +1,34 @@
+package config
+
+import (
+	"sync"
+
+	"github.com/hashicorp/terraform/helper/multierror"
+)
+
+// exprErrors are the errors built up from parsing. These should not
+// be accessed directly.
+var exprErrors []error
+var exprLock sync.Mutex
+var exprResult Interpolation
+
+// ExprParse parses the given expression and returns an executable
+// Interpolation.
+func ExprParse(v string) (Interpolation, error) {
+	exprLock.Lock()
+	defer exprLock.Unlock()
+	exprErrors = nil
+	exprResult = nil
+
+	// Parse
+	exprParse(&exprLex{input: v})
+
+	// Build up the errors
+	var err error
+	if len(exprErrors) > 0 {
+		err = &multierror.Error{Errors: exprErrors}
+		exprResult = nil
+	}
+
+	return exprResult, err
+}

config/expr_parse_test.go

@@ -0,0 +1,123 @@
+package config
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestExprParse(t *testing.T) {
+	cases := []struct {
+		Input  string
+		Result Interpolation
+		Error  bool
+	}{
+		{
+			"foo",
+			nil,
+			true,
+		},
+
+		{
+			`"foo"`,
+			&LiteralInterpolation{Literal: "foo"},
+			false,
+		},
+
+		{
+			"var.foo",
+			&VariableInterpolation{
+				Variable: &UserVariable{
+					Name: "foo",
+					key:  "var.foo",
+				},
+			},
+			false,
+		},
+
+		{
+			"lookup(var.foo, var.bar)",
+			&FunctionInterpolation{
+				Func: nil, // Funcs["lookup"]
+				Args: []Interpolation{
+					&VariableInterpolation{
+						Variable: &UserVariable{
+							Name: "foo",
+							key:  "var.foo",
+						},
+					},
+					&VariableInterpolation{
+						Variable: &UserVariable{
+							Name: "bar",
+							key:  "var.bar",
+						},
+					},
+				},
+			},
+			false,
+		},
+
+		{
+			"lookup(var.foo, lookup(var.baz, var.bar))",
+			&FunctionInterpolation{
+				Func: nil, // Funcs["lookup"]
+				Args: []Interpolation{
+					&VariableInterpolation{
+						Variable: &UserVariable{
+							Name: "foo",
+							key:  "var.foo",
+						},
+					},
+					&FunctionInterpolation{
+						Func: nil, // Funcs["lookup"]
+						Args: []Interpolation{
+							&VariableInterpolation{
+								Variable: &UserVariable{
+									Name: "baz",
+									key:  "var.baz",
+								},
+							},
+							&VariableInterpolation{
+								Variable: &UserVariable{
+									Name: "bar",
+									key:  "var.bar",
+								},
+							},
+						},
+					},
+				},
+			},
+			false,
+		},
+	}
+
+	for i, tc := range cases {
+		actual, err := ExprParse(tc.Input)
+		if (err != nil) != tc.Error {
+			t.Fatalf("%d. Error: %s", i, err)
+		}
+
+		// This is jank, but reflect.DeepEqual never has functions
+		// being the same.
+		f, ok := actual.(*FunctionInterpolation)
+		if ok {
+			fs := make([]*FunctionInterpolation, 1)
+			fs[0] = f
+			for len(fs) > 0 {
+				f := fs[0]
+				fs = fs[1:]
+
+				f.Func = nil
+				for _, a := range f.Args {
+					f, ok := a.(*FunctionInterpolation)
+					if ok {
+						fs = append(fs, f)
+					}
+				}
+			}
+		}
+
+		if !reflect.DeepEqual(actual, tc.Result) {
+			t.Fatalf("%d bad: %#v", i, actual)
+		}
+	}
+}

config/interpolate.go

@@ -17,7 +17,6 @@ var funcRegexp *regexp.Regexp = regexp.MustCompile(
 // Interpolations might be simple variable references, or it might be
 // function calls, or even nested function calls.
 type Interpolation interface {
-	FullString() string
 	Interpolate(map[string]string) (string, error)
 	Variables() map[string]InterpolatedVariable
 }
@@ -38,17 +37,19 @@ type InterpolatedVariable interface {
 // with some variable number of arguments to generate a value.
 type FunctionInterpolation struct {
 	Func InterpolationFunc
-	Args []InterpolatedVariable
+	Args []Interpolation
+}
 
-	key string
+// LiteralInterpolation implements Interpolation for literals. Ex:
+// ${"foo"} will equal "foo".
+type LiteralInterpolation struct {
+	Literal string
 }
 
 // VariableInterpolation implements Interpolation for simple variable
 // interpolation. Ex: "${var.foo}" or "${aws_instance.foo.bar}"
 type VariableInterpolation struct {
 	Variable InterpolatedVariable
-
-	key string
 }
 
 // A ResourceVariable is a variable that is referencing the field
@@ -74,61 +75,6 @@ type UserVariable struct {
 	key string
 }
 
-// NewInterpolation takes some string and returns the valid
-// Interpolation associated with it, or error if a valid
-// interpolation could not be found or the interpolation itself
-// is invalid.
-func NewInterpolation(v string) (Interpolation, error) {
-	match := funcRegexp.FindStringSubmatch(v)
-	if match != nil {
-		fn, ok := Funcs[match[1]]
-		if !ok {
-			return nil, fmt.Errorf(
-				"%s: Unknown function '%s'",
-				v, match[1])
-		}
-
-		args := make([]InterpolatedVariable, 0, len(match)-2)
-		for i := 2; i < len(match); i++ {
-			// This can be empty if we have a single argument
-			// due to the format of the regexp.
-			if match[i] == "" {
-				continue
-			}
-
-			v, err := NewInterpolatedVariable(match[i])
-			if err != nil {
-				return nil, err
-			}
-
-			args = append(args, v)
-		}
-
-		return &FunctionInterpolation{
-			Func: fn,
-			Args: args,
-
-			key: v,
-		}, nil
-	}
-
-	if idx := strings.Index(v, "."); idx >= 0 {
-		v, err := NewInterpolatedVariable(v)
-		if err != nil {
-			return nil, err
-		}
-
-		return &VariableInterpolation{
-			Variable: v,
-			key:      v.FullKey(),
-		}, nil
-	}
-
-	return nil, fmt.Errorf(
-		"Interpolation '%s' is not a valid interpolation. " +
-			"Please check your syntax and try again.")
-}
-
 func NewInterpolatedVariable(v string) (InterpolatedVariable, error) {
 	if !strings.HasPrefix(v, "var.") {
 		return NewResourceVariable(v)
@@ -137,21 +83,13 @@ func NewInterpolatedVariable(v string) (InterpolatedVariable, error) {
 	return NewUserVariable(v)
 }
 
-func (i *FunctionInterpolation) FullString() string {
-	return i.key
-}
-
 func (i *FunctionInterpolation) Interpolate(
 	vs map[string]string) (string, error) {
 	args := make([]string, len(i.Args))
 	for idx, a := range i.Args {
-		k := a.FullKey()
+		v, err := a.Interpolate(vs)
-		v, ok := vs[k]
+		if err != nil {
-		if !ok {
+			return "", err
-			return "", fmt.Errorf(
-				"%s: variable argument value unknown: %s",
-				i.FullString(),
-				k)
 		}
 
 		args[idx] = v
@@ -160,38 +98,48 @@ func (i *FunctionInterpolation) Interpolate(
 	return i.Func(vs, args...)
 }
 
+func (i *FunctionInterpolation) GoString() string {
+	return fmt.Sprintf("*%#v", *i)
+}
+
 func (i *FunctionInterpolation) Variables() map[string]InterpolatedVariable {
 	result := make(map[string]InterpolatedVariable)
 	for _, a := range i.Args {
-		k := a.FullKey()
+		for k, v := range a.Variables() {
-		if _, ok := result[k]; ok {
+			result[k] = v
-			continue
 		}
-
-		result[k] = a
 	}
 
 	return result
 }
 
-func (i *VariableInterpolation) FullString() string {
+func (i *LiteralInterpolation) Interpolate(
-	return i.key
+	map[string]string) (string, error) {
+	return i.Literal, nil
+}
+
+func (i *LiteralInterpolation) Variables() map[string]InterpolatedVariable {
+	return nil
 }
 
 func (i *VariableInterpolation) Interpolate(
 	vs map[string]string) (string, error) {
-	v, ok := vs[i.key]
+	v, ok := vs[i.Variable.FullKey()]
 	if !ok {
 		return "", fmt.Errorf(
 			"%s: value for variable not found",
-			i.key)
+			i.Variable.FullKey())
 	}
 
 	return v, nil
 }
 
+func (i *VariableInterpolation) GoString() string {
+	return fmt.Sprintf("*%#v", *i)
+}
+
 func (i *VariableInterpolation) Variables() map[string]InterpolatedVariable {
-	return map[string]InterpolatedVariable{i.key: i.Variable}
+	return map[string]InterpolatedVariable{i.Variable.FullKey(): i.Variable}
 }
 
 func NewResourceVariable(key string) (*ResourceVariable, error) {

config/interpolate_test.go

@@ -6,68 +6,6 @@ import (
 	"testing"
 )
 
-func TestNewInterpolation(t *testing.T) {
-	cases := []struct {
-		Input  string
-		Result Interpolation
-		Error  bool
-	}{
-		{
-			"foo",
-			nil,
-			true,
-		},
-
-		{
-			"var.foo",
-			&VariableInterpolation{
-				Variable: &UserVariable{
-					Name: "foo",
-					key:  "var.foo",
-				},
-				key: "var.foo",
-			},
-			false,
-		},
-
-		{
-			"lookup(var.foo, var.bar)",
-			&FunctionInterpolation{
-				Func: nil, // Funcs["lookup"]
-				Args: []InterpolatedVariable{
-					&UserVariable{
-						Name: "foo",
-						key:  "var.foo",
-					},
-					&UserVariable{
-						Name: "bar",
-						key:  "var.bar",
-					},
-				},
-				key: "lookup(var.foo, var.bar)",
-			},
-			false,
-		},
-	}
-
-	for i, tc := range cases {
-		actual, err := NewInterpolation(tc.Input)
-		if (err != nil) != tc.Error {
-			t.Fatalf("%d. Error: %s", i, err)
-		}
-
-		// This is jank, but reflect.DeepEqual never has functions
-		// being the same.
-		if f, ok := actual.(*FunctionInterpolation); ok {
-			f.Func = nil
-		}
-
-		if !reflect.DeepEqual(actual, tc.Result) {
-			t.Fatalf("%d bad: %#v", i, actual)
-		}
-	}
-}
-
 func TestNewInterpolatedVariable(t *testing.T) {
 	cases := []struct {
 		Input  string
@@ -171,11 +109,10 @@ func TestFunctionInterpolation(t *testing.T) {
 
 	i := &FunctionInterpolation{
 		Func: fn,
-		Args: []InterpolatedVariable{v1, v2},
+		Args: []Interpolation{
-		key:  "foo",
+			&VariableInterpolation{Variable: v1},
-	}
+			&VariableInterpolation{Variable: v2},
-	if i.FullString() != "foo" {
+		},
-		t.Fatalf("err: %#v", i)
 	}
 
 	expected := map[string]InterpolatedVariable{
@@ -199,6 +136,29 @@ func TestFunctionInterpolation(t *testing.T) {
 	}
 }
 
+func TestLiteralInterpolation_impl(t *testing.T) {
+	var _ Interpolation = new(LiteralInterpolation)
+}
+
+func TestLiteralInterpolation(t *testing.T) {
+	i := &LiteralInterpolation{
+		Literal: "bar",
+	}
+
+	if i.Variables() != nil {
+		t.Fatalf("bad: %#v", i.Variables())
+	}
+
+	actual, err := i.Interpolate(nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if actual != "bar" {
+		t.Fatalf("bad: %#v", actual)
+	}
+}
+
 func TestResourceVariable_impl(t *testing.T) {
 	var _ InterpolatedVariable = new(ResourceVariable)
 }
@@ -265,10 +225,7 @@ func TestVariableInterpolation(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 
-	i := &VariableInterpolation{Variable: uv, key: "var.foo"}
+	i := &VariableInterpolation{Variable: uv}
-	if i.FullString() != "var.foo" {
-		t.Fatalf("err: %#v", i)
-	}
 
 	expected := map[string]InterpolatedVariable{"var.foo": uv}
 	if !reflect.DeepEqual(i.Variables(), expected) {
@@ -293,7 +250,7 @@ func TestVariableInterpolation_missing(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 
-	i := &VariableInterpolation{Variable: uv, key: "var.foo"}
+	i := &VariableInterpolation{Variable: uv}
 	_, err = i.Interpolate(map[string]string{
 		"var.bar": "bar",
 	})

config/interpolate_walk.go

@@ -95,7 +95,7 @@ func (w *interpolationWalker) Primitive(v reflect.Value) error {
 		// Interpolation found, instantiate it
 		key := match[2]
 
-		i, err := NewInterpolation(key)
+		i, err := ExprParse(key)
 		if err != nil {
 			return err
 		}

config/interpolate_walk_test.go

@@ -29,7 +29,6 @@ func TestInterpolationWalker_detect(t *testing.T) {
 						Name: "foo",
 						key:  "var.foo",
 					},
-					key: "var.foo",
 				},
 			},
 		},
@@ -41,13 +40,14 @@ func TestInterpolationWalker_detect(t *testing.T) {
 			Result: []Interpolation{
 				&FunctionInterpolation{
 					Func: nil,
-					Args: []InterpolatedVariable{
+					Args: []Interpolation{
-						&UserVariable{
+						&VariableInterpolation{
+							Variable: &UserVariable{
 								Name: "foo",
 								key:  "var.foo",
 							},
 						},
-					key: "lookup(var.foo)",
+					},
 				},
 			},
 		},