configs: Experimental support for optional object type attributes

This builds on an experimental feature in the underlying cty library which
allows marking specific attribtues of an object type constraint as
optional, which in turn modifies how the cty conversion package handles
missing attributes in a source value: it will silently substitute a null
value of the appropriate type rather than returning an error.

In order to implement the experiment this commit temporarily forks the
HCL typeexpr extension package into a local internal/typeexpr package,
where I've extended the type constraint syntax to allow annotating object
type attributes as being optional using the HCL function call syntax.
If the experiment is successful -- both at the Terraform layer and in
the underlying cty library -- we'll likely send these modifications to
upstream HCL so that other HCL-based languages can potentially benefit
from this new capability.

Because it's experimental, the optional attribute modifier is allowed only
with an explicit opt-in to the module_variable_optional_attrs experiment.
This commit is contained in:
Martin Atkins 2020-09-27 17:02:54 -07:00
parent 18d59d768f
commit 0bbbb9c64b
11 changed files with 690 additions and 2 deletions

View File

@ -5,6 +5,7 @@ import (
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/terraform/experiments"
"github.com/zclconf/go-cty/cty"
)
// sniffActiveExperiments does minimal parsing of the given body for
@ -139,5 +140,51 @@ func checkModuleExperiments(m *Module) hcl.Diagnostics {
}
*/
if !m.ActiveExperiments.Has(experiments.ModuleVariableOptionalAttrs) {
for _, v := range m.Variables {
if typeConstraintHasOptionalAttrs(v.Type) {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Optional object type attributes are experimental",
Detail: "This feature is currently an opt-in experiment, subject to change in future releases based on feedback.\n\nActivate the feature for this module by adding module_variable_optional_attrs to the list of active experiments.",
Subject: v.DeclRange.Ptr(),
})
}
}
}
return diags
}
func typeConstraintHasOptionalAttrs(ty cty.Type) bool {
if ty == cty.NilType {
// Weird, but we'll just ignore it to avoid crashing.
return false
}
switch {
case ty.IsPrimitiveType():
return false
case ty.IsCollectionType():
return typeConstraintHasOptionalAttrs(ty.ElementType())
case ty.IsObjectType():
if len(ty.OptionalAttributes()) != 0 {
return true
}
for _, aty := range ty.AttributeTypes() {
if typeConstraintHasOptionalAttrs(aty) {
return true
}
}
return false
case ty.IsTupleType():
for _, ety := range ty.TupleElementTypes() {
if typeConstraintHasOptionalAttrs(ety) {
return true
}
}
return false
default:
return false
}
}

View File

@ -5,13 +5,13 @@ import (
"unicode"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/ext/typeexpr"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/convert"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/internal/typeexpr"
)
// A consistent detail message for all "not a valid identifier" diagnostics.

View File

@ -0,0 +1,6 @@
variable "a" {
type = object({
# The optional attributes experiment isn't enabled, so this isn't allowed.
a = optional(string)
})
}

View File

@ -0,0 +1,35 @@
terraform {
experiments = [
module_variable_optional_attrs, # WARNING: Experimental feature "module_variable_optional_attrs" is active
]
}
variable "a" {
type = object({
foo = optional(string)
})
}
variable "b" {
type = list(
object({
foo = optional(string)
})
)
}
variable "c" {
type = set(
object({
foo = optional(string)
})
)
}
variable "d" {
type = map(
object({
foo = optional(string)
})
)
}

View File

@ -14,12 +14,14 @@ type Experiment string
// identifier so that it can be specified in configuration.
const (
VariableValidation = Experiment("variable_validation")
ModuleVariableOptionalAttrs = Experiment("module_variable_optional_attrs")
)
func init() {
// Each experiment constant defined above must be registered here as either
// a current or a concluded experiment.
registerConcludedExperiment(VariableValidation, "Custom variable validation can now be used by default, without enabling an experiment.")
registerCurrentExperiment(ModuleVariableOptionalAttrs)
}
// GetCurrent takes an experiment name and returns the experiment value

10
internal/typeexpr/doc.go Normal file
View File

@ -0,0 +1,10 @@
// Package typeexpr is a fork of github.com/hashicorp/hcl/v2/ext/typeexpr
// which has additional experimental support for optional attributes.
//
// This is here as part of the module_variable_optional_attrs experiment.
// If that experiment is successful, the changes here may be upstreamed into
// HCL itself or, if we deem it to be Terraform-specific, we should at least
// update this documentation to reflect that this is now the primary
// Terraform-specific type expression implementation, separate from the
// upstream HCL one.
package typeexpr

View File

@ -0,0 +1,250 @@
package typeexpr
import (
"fmt"
"github.com/hashicorp/hcl/v2"
"github.com/zclconf/go-cty/cty"
)
const invalidTypeSummary = "Invalid type specification"
// getType is the internal implementation of both Type and TypeConstraint,
// using the passed flag to distinguish. When constraint is false, the "any"
// keyword will produce an error.
func getType(expr hcl.Expression, constraint bool) (cty.Type, hcl.Diagnostics) {
// First we'll try for one of our keywords
kw := hcl.ExprAsKeyword(expr)
switch kw {
case "bool":
return cty.Bool, nil
case "string":
return cty.String, nil
case "number":
return cty.Number, nil
case "any":
if constraint {
return cty.DynamicPseudoType, nil
}
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("The keyword %q cannot be used in this type specification: an exact type is required.", kw),
Subject: expr.Range().Ptr(),
}}
case "list", "map", "set":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("The %s type constructor requires one argument specifying the element type.", kw),
Subject: expr.Range().Ptr(),
}}
case "object":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "The object type constructor requires one argument specifying the attribute types and values as a map.",
Subject: expr.Range().Ptr(),
}}
case "tuple":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "The tuple type constructor requires one argument specifying the element types as a list.",
Subject: expr.Range().Ptr(),
}}
case "":
// okay! we'll fall through and try processing as a call, then.
default:
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("The keyword %q is not a valid type specification.", kw),
Subject: expr.Range().Ptr(),
}}
}
// If we get down here then our expression isn't just a keyword, so we'll
// try to process it as a call instead.
call, diags := hcl.ExprCall(expr)
if diags.HasErrors() {
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "A type specification is either a primitive type keyword (bool, number, string) or a complex type constructor call, like list(string).",
Subject: expr.Range().Ptr(),
}}
}
switch call.Name {
case "bool", "string", "number", "any":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("Primitive type keyword %q does not expect arguments.", call.Name),
Subject: &call.ArgsRange,
}}
}
if len(call.Arguments) != 1 {
contextRange := call.ArgsRange
subjectRange := call.ArgsRange
if len(call.Arguments) > 1 {
// If we have too many arguments (as opposed to too _few_) then
// we'll highlight the extraneous arguments as the diagnostic
// subject.
subjectRange = hcl.RangeBetween(call.Arguments[1].Range(), call.Arguments[len(call.Arguments)-1].Range())
}
switch call.Name {
case "list", "set", "map":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("The %s type constructor requires one argument specifying the element type.", call.Name),
Subject: &subjectRange,
Context: &contextRange,
}}
case "object":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "The object type constructor requires one argument specifying the attribute types and values as a map.",
Subject: &subjectRange,
Context: &contextRange,
}}
case "tuple":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "The tuple type constructor requires one argument specifying the element types as a list.",
Subject: &subjectRange,
Context: &contextRange,
}}
}
}
switch call.Name {
case "list":
ety, diags := getType(call.Arguments[0], constraint)
return cty.List(ety), diags
case "set":
ety, diags := getType(call.Arguments[0], constraint)
return cty.Set(ety), diags
case "map":
ety, diags := getType(call.Arguments[0], constraint)
return cty.Map(ety), diags
case "object":
attrDefs, diags := hcl.ExprMap(call.Arguments[0])
if diags.HasErrors() {
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "Object type constructor requires a map whose keys are attribute names and whose values are the corresponding attribute types.",
Subject: call.Arguments[0].Range().Ptr(),
Context: expr.Range().Ptr(),
}}
}
atys := make(map[string]cty.Type)
var optAttrs []string
for _, attrDef := range attrDefs {
attrName := hcl.ExprAsKeyword(attrDef.Key)
if attrName == "" {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "Object constructor map keys must be attribute names.",
Subject: attrDef.Key.Range().Ptr(),
Context: expr.Range().Ptr(),
})
continue
}
atyExpr := attrDef.Value
// the attribute type expression might be wrapped in the special
// modifier optional(...) to indicate an optional attribute. If
// so, we'll unwrap that first and make a note about it being
// optional for when we construct the type below.
if call, diags := hcl.ExprCall(atyExpr); !diags.HasErrors() {
if call.Name == "optional" {
if len(call.Arguments) < 1 {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "Optional attribute modifier requires the attribute type as its argument.",
Subject: call.ArgsRange.Ptr(),
Context: atyExpr.Range().Ptr(),
})
continue
}
if constraint {
if len(call.Arguments) > 1 {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "Optional attribute modifier expects only one argument: the attribute type.",
Subject: call.ArgsRange.Ptr(),
Context: atyExpr.Range().Ptr(),
})
}
optAttrs = append(optAttrs, attrName)
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "Optional attribute modifier is only for type constraints, not for exact types.",
Subject: call.NameRange.Ptr(),
Context: atyExpr.Range().Ptr(),
})
}
atyExpr = call.Arguments[0]
}
}
aty, attrDiags := getType(atyExpr, constraint)
diags = append(diags, attrDiags...)
atys[attrName] = aty
}
// NOTE: ObjectWithOptionalAttrs is experimental in cty at the
// time of writing, so this interface might change even in future
// minor versions of cty. We're accepting that because Terraform
// itself is considering optional attributes as experimental right now.
return cty.ObjectWithOptionalAttrs(atys, optAttrs), diags
case "tuple":
elemDefs, diags := hcl.ExprList(call.Arguments[0])
if diags.HasErrors() {
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: "Tuple type constructor requires a list of element types.",
Subject: call.Arguments[0].Range().Ptr(),
Context: expr.Range().Ptr(),
}}
}
etys := make([]cty.Type, len(elemDefs))
for i, defExpr := range elemDefs {
ety, elemDiags := getType(defExpr, constraint)
diags = append(diags, elemDiags...)
etys[i] = ety
}
return cty.Tuple(etys), diags
case "optional":
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("Keyword %q is valid only as a modifier for object type attributes.", call.Name),
Subject: call.NameRange.Ptr(),
}}
default:
// Can't access call.Arguments in this path because we've not validated
// that it contains exactly one expression here.
return cty.DynamicPseudoType, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: invalidTypeSummary,
Detail: fmt.Sprintf("Keyword %q is not a valid type constructor.", call.Name),
Subject: expr.Range().Ptr(),
}}
}
}

129
internal/typeexpr/public.go Normal file
View File

@ -0,0 +1,129 @@
package typeexpr
import (
"bytes"
"fmt"
"sort"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/hashicorp/hcl/v2"
"github.com/zclconf/go-cty/cty"
)
// Type attempts to process the given expression as a type expression and, if
// successful, returns the resulting type. If unsuccessful, error diagnostics
// are returned.
func Type(expr hcl.Expression) (cty.Type, hcl.Diagnostics) {
return getType(expr, false)
}
// TypeConstraint attempts to parse the given expression as a type constraint
// and, if successful, returns the resulting type. If unsuccessful, error
// diagnostics are returned.
//
// A type constraint has the same structure as a type, but it additionally
// allows the keyword "any" to represent cty.DynamicPseudoType, which is often
// used as a wildcard in type checking and type conversion operations.
func TypeConstraint(expr hcl.Expression) (cty.Type, hcl.Diagnostics) {
return getType(expr, true)
}
// TypeString returns a string rendering of the given type as it would be
// expected to appear in the HCL native syntax.
//
// This is primarily intended for showing types to the user in an application
// that uses typexpr, where the user can be assumed to be familiar with the
// type expression syntax. In applications that do not use typeexpr these
// results may be confusing to the user and so type.FriendlyName may be
// preferable, even though it's less precise.
//
// TypeString produces reasonable results only for types like what would be
// produced by the Type and TypeConstraint functions. In particular, it cannot
// support capsule types.
func TypeString(ty cty.Type) string {
// Easy cases first
switch ty {
case cty.String:
return "string"
case cty.Bool:
return "bool"
case cty.Number:
return "number"
case cty.DynamicPseudoType:
return "any"
}
if ty.IsCapsuleType() {
panic("TypeString does not support capsule types")
}
if ty.IsCollectionType() {
ety := ty.ElementType()
etyString := TypeString(ety)
switch {
case ty.IsListType():
return fmt.Sprintf("list(%s)", etyString)
case ty.IsSetType():
return fmt.Sprintf("set(%s)", etyString)
case ty.IsMapType():
return fmt.Sprintf("map(%s)", etyString)
default:
// Should never happen because the above is exhaustive
panic("unsupported collection type")
}
}
if ty.IsObjectType() {
var buf bytes.Buffer
buf.WriteString("object({")
atys := ty.AttributeTypes()
names := make([]string, 0, len(atys))
for name := range atys {
names = append(names, name)
}
sort.Strings(names)
first := true
for _, name := range names {
aty := atys[name]
if !first {
buf.WriteByte(',')
}
if !hclsyntax.ValidIdentifier(name) {
// Should never happen for any type produced by this package,
// but we'll do something reasonable here just so we don't
// produce garbage if someone gives us a hand-assembled object
// type that has weird attribute names.
// Using Go-style quoting here isn't perfect, since it doesn't
// exactly match HCL syntax, but it's fine for an edge-case.
buf.WriteString(fmt.Sprintf("%q", name))
} else {
buf.WriteString(name)
}
buf.WriteByte('=')
buf.WriteString(TypeString(aty))
first = false
}
buf.WriteString("})")
return buf.String()
}
if ty.IsTupleType() {
var buf bytes.Buffer
buf.WriteString("tuple([")
etys := ty.TupleElementTypes()
first := true
for _, ety := range etys {
if !first {
buf.WriteByte(',')
}
buf.WriteString(TypeString(ety))
first = false
}
buf.WriteString("])")
return buf.String()
}
// Should never happen because we covered all cases above.
panic(fmt.Errorf("unsupported type %#v", ty))
}

View File

@ -0,0 +1,118 @@
package typeexpr
import (
"fmt"
"reflect"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/ext/customdecode"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/convert"
"github.com/zclconf/go-cty/cty/function"
)
// TypeConstraintType is a cty capsule type that allows cty type constraints to
// be used as values.
//
// If TypeConstraintType is used in a context supporting the
// customdecode.CustomExpressionDecoder extension then it will implement
// expression decoding using the TypeConstraint function, thus allowing
// type expressions to be used in contexts where value expressions might
// normally be expected, such as in arguments to function calls.
var TypeConstraintType cty.Type
// TypeConstraintVal constructs a cty.Value whose type is
// TypeConstraintType.
func TypeConstraintVal(ty cty.Type) cty.Value {
return cty.CapsuleVal(TypeConstraintType, &ty)
}
// TypeConstraintFromVal extracts the type from a cty.Value of
// TypeConstraintType that was previously constructed using TypeConstraintVal.
//
// If the given value isn't a known, non-null value of TypeConstraintType
// then this function will panic.
func TypeConstraintFromVal(v cty.Value) cty.Type {
if !v.Type().Equals(TypeConstraintType) {
panic("value is not of TypeConstraintType")
}
ptr := v.EncapsulatedValue().(*cty.Type)
return *ptr
}
// ConvertFunc is a cty function that implements type conversions.
//
// Its signature is as follows:
// convert(value, type_constraint)
//
// ...where type_constraint is a type constraint expression as defined by
// typeexpr.TypeConstraint.
//
// It relies on HCL's customdecode extension and so it's not suitable for use
// in non-HCL contexts or if you are using a HCL syntax implementation that
// does not support customdecode for function arguments. However, it _is_
// supported for function calls in the HCL native expression syntax.
var ConvertFunc function.Function
func init() {
TypeConstraintType = cty.CapsuleWithOps("type constraint", reflect.TypeOf(cty.Type{}), &cty.CapsuleOps{
ExtensionData: func(key interface{}) interface{} {
switch key {
case customdecode.CustomExpressionDecoder:
return customdecode.CustomExpressionDecoderFunc(
func(expr hcl.Expression, ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
ty, diags := TypeConstraint(expr)
if diags.HasErrors() {
return cty.NilVal, diags
}
return TypeConstraintVal(ty), nil
},
)
default:
return nil
}
},
TypeGoString: func(_ reflect.Type) string {
return "typeexpr.TypeConstraintType"
},
GoString: func(raw interface{}) string {
tyPtr := raw.(*cty.Type)
return fmt.Sprintf("typeexpr.TypeConstraintVal(%#v)", *tyPtr)
},
RawEquals: func(a, b interface{}) bool {
aPtr := a.(*cty.Type)
bPtr := b.(*cty.Type)
return (*aPtr).Equals(*bPtr)
},
})
ConvertFunc = function.New(&function.Spec{
Params: []function.Parameter{
{
Name: "value",
Type: cty.DynamicPseudoType,
AllowNull: true,
AllowDynamicType: true,
},
{
Name: "type",
Type: TypeConstraintType,
},
},
Type: func(args []cty.Value) (cty.Type, error) {
wantTypePtr := args[1].EncapsulatedValue().(*cty.Type)
got, err := convert.Convert(args[0], *wantTypePtr)
if err != nil {
return cty.NilType, function.NewArgError(0, err)
}
return got.Type(), nil
},
Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
v, err := convert.Convert(args[0], retType)
if err != nil {
return cty.NilVal, function.NewArgError(0, err)
}
return v, nil
},
})
}

View File

@ -11995,3 +11995,57 @@ resource "test_resource" "foo" {
t.Fatalf("wrong number of sensitive paths, expected 0, got, %v", len(fooState2.Current.AttrSensitivePaths))
}
}
func TestContext2Apply_moduleVariableOptionalAttributes(t *testing.T) {
m := testModuleInline(t, map[string]string{
"main.tf": `
terraform {
experiments = [module_variable_optional_attrs]
}
variable "in" {
type = object({
required = string
optional = optional(string)
})
}
output "out" {
value = var.in
}
`})
ctx := testContext2(t, &ContextOpts{
Variables: InputValues{
"in": &InputValue{
Value: cty.MapVal(map[string]cty.Value{
"required": cty.StringVal("boop"),
}),
SourceType: ValueFromCaller,
},
},
Config: m,
})
_, diags := ctx.Plan()
if diags.HasErrors() {
t.Fatal(diags.ErrWithWarnings())
}
state, diags := ctx.Apply()
if diags.HasErrors() {
t.Fatal(diags.ErrWithWarnings())
}
got := state.RootModule().OutputValues["out"].Value
want := cty.ObjectVal(map[string]cty.Value{
"required": cty.StringVal("boop"),
// Because "optional" was marked as optional, it got silently filled
// in as a null value of string type rather than returning an error.
"optional": cty.NullVal(cty.String),
})
if !want.RawEquals(got) {
t.Fatalf("wrong result\ngot: %#v\nwant: %#v", got, want)
}
}

View File

@ -264,3 +264,40 @@ variable "no_type_constraint" {
In this case, Terraform will replace `any` with the exact type of the given
value and thus perform no type conversion whatsoever.
## Experimental: Optional Object Type Attributes
From Terraform v0.14 there is _experimental_ support for marking particular
attributes as optional in an object type constraint.
To mark an attribute as optional, use the additional `optional(...)` modifier
around its type declaration:
```hcl
variable "with_optional_attribute" {
type = object({
a = string # a required attribute
b = optional(string) # an optional attribute
})
}
```
By default, for required attributes, Terraform will return an error if the
source value has no matching attribute. Marking an attribute as optional
changes the behavior in that situation: Terraform will instead just silently
insert `null` as the value of the attribute, allowing the recieving module
to describe an appropriate fallback behavior.
Because this feature is currently experimental, it requires an explicit
opt-in on a per-module basis. To use it, write a `terraform` block with the
`experiments` argument set as follows:
```hcl
terraform {
experiments = [module_variable_optional_attrs]
}
```
Until the experiment is concluded, the behavior of this feature may see
breaking changes even in minor releases. We recommend using this feature
only in prerelease versions of modules as long as it remains experimental.