hcl2shim: Handle unknown values when shimming to/from flatmap

Previously unknown values were round-tripping through flatmap and coming
out as known strings containing the UnknownVariableValue. (The classic bug
that, ironically, was one of the big reasons to write cty!)

Now we properly handle unknown values in both directions: going in to
flatmap we write UnknownVariableValue at the appropriate key (as the count
for sequences or maps) and then coming out of flatmap we turn
UnknownVariableValue back into a cty unknown value of the requested type.

config/hcl2shim/flatmap.go

@@ -64,6 +64,20 @@ func flatmapValueFromHCL2Primitive(m map[string]string, key string, val cty.Valu
 }
 
 func flatmapValueFromHCL2Map(m map[string]string, prefix string, val cty.Value) {
+	if !val.IsKnown() {
+		switch {
+		case val.Type().IsObjectType():
+			// Whole objects can't be unknown in flatmap, so instead we'll
+			// just write all of the attribute values out as unknown.
+			for name, aty := range val.Type().AttributeTypes() {
+				flatmapValueFromHCL2Value(m, prefix+name, cty.UnknownVal(aty))
+			}
+		default:
+			m[prefix+"%"] = UnknownVariableValue
+		}
+		return
+	}
+
 	len := 0
 	for it := val.ElementIterator(); it.Next(); {
 		ak, av := it.Element()
@@ -77,6 +91,11 @@ func flatmapValueFromHCL2Map(m map[string]string, prefix string, val cty.Value)
 }
 
 func flatmapValueFromHCL2Seq(m map[string]string, prefix string, val cty.Value) {
+	if !val.IsKnown() {
+		m[prefix+"#"] = UnknownVariableValue
+		return
+	}
+
 	// For sets this won't actually generate exactly what helper/schema would've
 	// generated, because we don't have access to the set key function it
 	// would've used. However, in practice it doesn't actually matter what the
@@ -150,6 +169,9 @@ func hcl2ValueFromFlatmapPrimitive(m map[string]string, key string, ty cty.Type)
 	if !exists {
 		return cty.NullVal(ty), nil
 	}
+	if rawVal == UnknownVariableValue {
+		return cty.UnknownVal(ty), nil
+	}
 
 	var err error
 	val := cty.StringVal(rawVal)
@@ -182,6 +204,10 @@ func hcl2ValueFromFlatmapTuple(m map[string]string, prefix string, etys []cty.Ty
 	if !exists {
 		return cty.NullVal(cty.Tuple(etys)), nil
 	}
+	if countStr == UnknownVariableValue {
+		return cty.UnknownVal(cty.Tuple(etys)), nil
+	}
+
 	count, err := strconv.Atoi(countStr)
 	if err != nil {
 		return cty.DynamicVal, fmt.Errorf("invalid count value for %q in state: %s", prefix, err)
@@ -209,8 +235,10 @@ func hcl2ValueFromFlatmapMap(m map[string]string, prefix string, ty cty.Type) (c
 	// We actually don't really care about the "count" of a map for our
 	// purposes here, but we do need to check if it _exists_ in order to
 	// recognize the difference between null (not set at all) and empty.
-	if _, exists := m[prefix+"%"]; !exists {
+	if strCount, exists := m[prefix+"%"]; !exists {
 		return cty.NullVal(ty), nil
+	} else if strCount == UnknownVariableValue {
+		return cty.UnknownVal(ty), nil
 	}
 
 	for fullKey := range m {
@@ -249,6 +277,10 @@ func hcl2ValueFromFlatmapList(m map[string]string, prefix string, ty cty.Type) (
 	if !exists {
 		return cty.NullVal(ty), nil
 	}
+	if countStr == UnknownVariableValue {
+		return cty.UnknownVal(ty), nil
+	}
+
 	count, err := strconv.Atoi(countStr)
 	if err != nil {
 		return cty.DynamicVal, fmt.Errorf("invalid count value for %q in state: %s", prefix, err)
@@ -279,8 +311,10 @@ func hcl2ValueFromFlatmapSet(m map[string]string, prefix string, ty cty.Type) (c
 	// We actually don't really care about the "count" of a set for our
 	// purposes here, but we do need to check if it _exists_ in order to
 	// recognize the difference between null (not set at all) and empty.
-	if _, exists := m[prefix+"#"]; !exists {
+	if strCount, exists := m[prefix+"#"]; !exists {
 		return cty.NullVal(ty), nil
+	} else if strCount == UnknownVariableValue {
+		return cty.UnknownVal(ty), nil
 	}
 
 	for fullKey := range m {

config/hcl2shim/flatmap_test.go

@@ -26,6 +26,14 @@ func TestFlatmapValueFromHCL2(t *testing.T) {
 				"foo": "hello",
 			},
 		},
+		{
+			cty.ObjectVal(map[string]cty.Value{
+				"foo": cty.UnknownVal(cty.Bool),
+			}),
+			map[string]string{
+				"foo": UnknownVariableValue,
+			},
+		},
 		{
 			cty.ObjectVal(map[string]cty.Value{
 				"foo": cty.NumberIntVal(12),
@@ -64,6 +72,14 @@ func TestFlatmapValueFromHCL2(t *testing.T) {
 				"foo.#": "0",
 			},
 		},
+		{
+			cty.ObjectVal(map[string]cty.Value{
+				"foo": cty.UnknownVal(cty.List(cty.String)),
+			}),
+			map[string]string{
+				"foo.#": UnknownVariableValue,
+			},
+		},
 		{
 			cty.ObjectVal(map[string]cty.Value{
 				"foo": cty.ListVal([]cty.Value{
@@ -101,6 +117,14 @@ func TestFlatmapValueFromHCL2(t *testing.T) {
 				"foo.hello.world": "10",
 			},
 		},
+		{
+			cty.ObjectVal(map[string]cty.Value{
+				"foo": cty.UnknownVal(cty.Map(cty.String)),
+			}),
+			map[string]string{
+				"foo.%": UnknownVariableValue,
+			},
+		},
 		{
 			cty.ObjectVal(map[string]cty.Value{
 				"foo": cty.MapVal(map[string]cty.Value{
@@ -127,6 +151,14 @@ func TestFlatmapValueFromHCL2(t *testing.T) {
 				"foo.1": "world",
 			},
 		},
+		{
+			cty.ObjectVal(map[string]cty.Value{
+				"foo": cty.UnknownVal(cty.Set(cty.Number)),
+			}),
+			map[string]string{
+				"foo.#": UnknownVariableValue,
+			},
+		},
 		{
 			cty.ObjectVal(map[string]cty.Value{
 				"foo": cty.ListVal([]cty.Value{
@@ -179,6 +211,23 @@ func TestFlatmapValueFromHCL2(t *testing.T) {
 				"foo.1.baz.1": "true",
 			},
 		},
+		{
+			cty.ObjectVal(map[string]cty.Value{
+				"foo": cty.ListVal([]cty.Value{
+					cty.UnknownVal(cty.Object(map[string]cty.Type{
+						"bar": cty.String,
+						"baz": cty.List(cty.Bool),
+						"bap": cty.Map(cty.Number),
+					})),
+				}),
+			}),
+			map[string]string{
+				"foo.#":       "1",
+				"foo.0.bar":   UnknownVariableValue,
+				"foo.0.baz.#": UnknownVariableValue,
+				"foo.0.bap.%": UnknownVariableValue,
+			},
+		},
 	}
 
 	for _, test := range tests {
@@ -216,16 +265,19 @@ func TestHCL2ValueFromFlatmap(t *testing.T) {
 				"foo": "blah",
 				"bar": "true",
 				"baz": "12.5",
+				"unk": UnknownVariableValue,
 			},
 			Type: cty.Object(map[string]cty.Type{
 				"foo": cty.String,
 				"bar": cty.Bool,
 				"baz": cty.Number,
+				"unk": cty.Bool,
 			}),
 			Want: cty.ObjectVal(map[string]cty.Value{
 				"foo": cty.StringVal("blah"),
 				"bar": cty.True,
 				"baz": cty.NumberFloatVal(12.5),
+				"unk": cty.UnknownVal(cty.Bool),
 			}),
 		},
 		{
@@ -239,6 +291,17 @@ func TestHCL2ValueFromFlatmap(t *testing.T) {
 				"foo": cty.ListValEmpty(cty.String),
 			}),
 		},
+		{
+			Flatmap: map[string]string{
+				"foo.#": UnknownVariableValue,
+			},
+			Type: cty.Object(map[string]cty.Type{
+				"foo": cty.List(cty.String),
+			}),
+			Want: cty.ObjectVal(map[string]cty.Value{
+				"foo": cty.UnknownVal(cty.List(cty.String)),
+			}),
+		},
 		{
 			Flatmap: map[string]string{
 				"foo.#": "1",
@@ -288,6 +351,23 @@ func TestHCL2ValueFromFlatmap(t *testing.T) {
 				}),
 			}),
 		},
+		{
+			Flatmap: map[string]string{
+				"foo.#": UnknownVariableValue,
+			},
+			Type: cty.Object(map[string]cty.Type{
+				"foo": cty.Tuple([]cty.Type{
+					cty.String,
+					cty.Bool,
+				}),
+			}),
+			Want: cty.ObjectVal(map[string]cty.Value{
+				"foo": cty.UnknownVal(cty.Tuple([]cty.Type{
+					cty.String,
+					cty.Bool,
+				})),
+			}),
+		},
 		{
 			Flatmap: map[string]string{
 				"foo.#": "0",
@@ -299,6 +379,17 @@ func TestHCL2ValueFromFlatmap(t *testing.T) {
 				"foo": cty.SetValEmpty(cty.String),
 			}),
 		},
+		{
+			Flatmap: map[string]string{
+				"foo.#": UnknownVariableValue,
+			},
+			Type: cty.Object(map[string]cty.Type{
+				"foo": cty.Set(cty.String),
+			}),
+			Want: cty.ObjectVal(map[string]cty.Value{
+				"foo": cty.UnknownVal(cty.Set(cty.String)),
+			}),
+		},
 		{
 			Flatmap: map[string]string{
 				"foo.#":        "1",
@@ -357,6 +448,17 @@ func TestHCL2ValueFromFlatmap(t *testing.T) {
 				}),
 			}),
 		},
+		{
+			Flatmap: map[string]string{
+				"foo.%": UnknownVariableValue,
+			},
+			Type: cty.Object(map[string]cty.Type{
+				"foo": cty.Map(cty.Bool),
+			}),
+			Want: cty.ObjectVal(map[string]cty.Value{
+				"foo": cty.UnknownVal(cty.Map(cty.Bool)),
+			}),
+		},
 		{
 			Flatmap: map[string]string{
 				"foo.#":     "2",