core: Opportunistic schema loading during graph construction

Previously the graph builders all expected to be given a full manifest
of all of the plugin component schemas that they could need during their
analysis work. That made sense when terraform.NewContext would always
proactively load all of the schemas before doing any other work, but we
now have a load-as-needed strategy for schemas.

We'll now have the graph builders use the contextPlugins object they each
already hold to retrieve individual schemas when needed. This avoids the
need to prepare a redundant data structure to pass alongside the
contextPlugins object, and leans on the memoization behavior inside
contextPlugins to preserve the old behavior of loading each provider's
schema only once.
This commit is contained in:
Martin Atkins 2021-08-31 16:36:27 -07:00
parent a59c2fe1b9
commit 38ec730b0e
21 changed files with 126 additions and 127 deletions

View File

@ -32,7 +32,7 @@ func (c *Context) Apply(plan *plans.Plan, config *configs.Config) (*states.State
log.Printf("[DEBUG] Building and walking apply graph for %s plan", plan.UIMode)
graph, operation, moreDiags := c.applyGraph(plan, config, schemas, true)
graph, operation, moreDiags := c.applyGraph(plan, config, true)
if moreDiags.HasErrors() {
return nil, diags
}
@ -90,13 +90,12 @@ Note that the -target option is not suitable for routine use, and is provided on
return newState, diags
}
func (c *Context) applyGraph(plan *plans.Plan, config *configs.Config, schemas *Schemas, validate bool) (*Graph, walkOperation, tfdiags.Diagnostics) {
func (c *Context) applyGraph(plan *plans.Plan, config *configs.Config, validate bool) (*Graph, walkOperation, tfdiags.Diagnostics) {
graph, diags := (&ApplyGraphBuilder{
Config: config,
Changes: plan.Changes,
State: plan.PriorState,
Plugins: c.plugins,
Schemas: schemas,
Targets: plan.TargetAddrs,
ForceReplace: plan.ForceReplaceAddrs,
Validate: validate,
@ -130,13 +129,7 @@ func (c *Context) ApplyGraphForUI(plan *plans.Plan, config *configs.Config) (*Gr
var diags tfdiags.Diagnostics
schemas, moreDiags := c.Schemas(config, plan.PriorState)
diags = diags.Append(moreDiags)
if diags.HasErrors() {
return nil, diags
}
graph, _, moreDiags := c.applyGraph(plan, config, schemas, false)
graph, _, moreDiags := c.applyGraph(plan, config, false)
diags = diags.Append(moreDiags)
return graph, diags
}

View File

@ -69,7 +69,6 @@ func (c *Context) Eval(config *configs.Config, state *states.State, moduleAddr a
Config: config,
State: state,
Plugins: c.plugins,
Schemas: schemas,
}).Build(addrs.RootModuleInstance)
diags = diags.Append(moreDiags)
if moreDiags.HasErrors() {

View File

@ -64,7 +64,6 @@ func (c *Context) Import(config *configs.Config, prevRunState *states.State, opt
ImportTargets: opts.Targets,
Config: config,
Plugins: c.plugins,
Schemas: schemas,
}
// Build the graph

View File

@ -372,7 +372,6 @@ func (c *Context) planGraph(config *configs.Config, prevRunState *states.State,
Config: config,
State: prevRunState,
Plugins: c.plugins,
Schemas: schemas,
Targets: opts.Targets,
ForceReplace: opts.ForceReplace,
Validate: validate,
@ -384,7 +383,6 @@ func (c *Context) planGraph(config *configs.Config, prevRunState *states.State,
Config: config,
State: prevRunState,
Plugins: c.plugins,
Schemas: schemas,
Targets: opts.Targets,
Validate: validate,
skipRefresh: opts.SkipRefresh,
@ -396,7 +394,6 @@ func (c *Context) planGraph(config *configs.Config, prevRunState *states.State,
Config: config,
State: prevRunState,
Plugins: c.plugins,
Schemas: schemas,
Targets: opts.Targets,
Validate: validate,
skipRefresh: opts.SkipRefresh,

View File

@ -134,6 +134,40 @@ func (cp *contextPlugins) ProviderSchema(addr addrs.Provider) (*ProviderSchema,
return s, nil
}
// ProviderConfigSchema is a helper wrapper around ProviderSchema which first
// reads the full schema of the given provider and then extracts just the
// provider's configuration schema, which defines what's expected in a
// "provider" block in the configuration when configuring this provider.
func (cp *contextPlugins) ProviderConfigSchema(providerAddr addrs.Provider) (*configschema.Block, error) {
providerSchema, err := cp.ProviderSchema(providerAddr)
if err != nil {
return nil, err
}
return providerSchema.Provider, nil
}
// ResourceTypeSchema is a helper wrapper around ProviderSchema which first
// reads the schema of the given provider and then tries to find the schema
// for the resource type of the given resource mode in that provider.
//
// ResourceTypeSchema will return an error if the provider schema lookup
// fails, but will return nil if the provider schema lookup succeeds but then
// the provider doesn't have a resource of the requested type.
//
// Managed resource types have versioned schemas, so the second return value
// is the current schema version number for the requested resource. The version
// is irrelevant for other resource modes.
func (cp *contextPlugins) ResourceTypeSchema(providerAddr addrs.Provider, resourceMode addrs.ResourceMode, resourceType string) (*configschema.Block, uint64, error) {
providerSchema, err := cp.ProviderSchema(providerAddr)
if err != nil {
return nil, 0, err
}
schema, version := providerSchema.SchemaForResourceType(resourceMode, resourceType)
return schema, version, nil
}
// ProvisionerSchema uses a temporary instance of the provisioner with the
// given type name to obtain the schema for that provisioner's configuration.
//

View File

@ -46,7 +46,6 @@ func (c *Context) Validate(config *configs.Config) tfdiags.Diagnostics {
graph, moreDiags := ValidateGraphBuilder(&PlanGraphBuilder{
Config: config,
Plugins: c.plugins,
Schemas: schemas,
Validate: true,
State: states.NewState(),
}).Build(addrs.RootModuleInstance)

View File

@ -1201,7 +1201,6 @@ func TestContext2Validate_PlanGraphBuilder(t *testing.T) {
Config: fixture.Config,
State: states.NewState(),
Plugins: c.plugins,
Schemas: schemas,
}).Build(addrs.RootModuleInstance)
if diags.HasErrors() {
t.Fatalf("errors from PlanGraphBuilder: %s", diags.Err())

View File

@ -30,10 +30,6 @@ type ApplyGraphBuilder struct {
// provisioners) available for use.
Plugins *contextPlugins
// Schemas is the repository of schemas we will draw from to analyse
// the configuration.
Schemas *Schemas
// Targets are resources to target. This is only required to make sure
// unnecessary outputs aren't included in the apply graph. The plan
// builder successfully handles targeting resources. In the future,
@ -123,7 +119,7 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer {
// Must attach schemas before ReferenceTransformer so that we can
// analyze the configuration to find references.
&AttachSchemaTransformer{Schemas: b.Schemas, Config: b.Config},
&AttachSchemaTransformer{Plugins: b.Plugins, Config: b.Config},
// Create expansion nodes for all of the module calls. This must
// come after all other transformers that create nodes representing
@ -142,12 +138,10 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer {
&DestroyEdgeTransformer{
Config: b.Config,
State: b.State,
Schemas: b.Schemas,
},
&CBDEdgeTransformer{
Config: b.Config,
State: b.State,
Schemas: b.Schemas,
},
// We need to remove configuration nodes that are not used at all, as

View File

@ -49,7 +49,6 @@ func TestApplyGraphBuilder(t *testing.T) {
Config: testModule(t, "graph-builder-apply-basic"),
Changes: changes,
Plugins: simpleMockPluginLibrary(),
Schemas: simpleTestSchemas(),
}
g, err := b.Build(addrs.RootModuleInstance)
@ -113,7 +112,6 @@ func TestApplyGraphBuilder_depCbd(t *testing.T) {
Config: testModule(t, "graph-builder-apply-dep-cbd"),
Changes: changes,
Plugins: simpleMockPluginLibrary(),
Schemas: simpleTestSchemas(),
State: state,
}
@ -187,7 +185,6 @@ func TestApplyGraphBuilder_doubleCBD(t *testing.T) {
Config: testModule(t, "graph-builder-apply-double-cbd"),
Changes: changes,
Plugins: simpleMockPluginLibrary(),
Schemas: simpleTestSchemas(),
}
g, err := b.Build(addrs.RootModuleInstance)
@ -282,7 +279,6 @@ func TestApplyGraphBuilder_destroyStateOnly(t *testing.T) {
Changes: changes,
State: state,
Plugins: simpleMockPluginLibrary(),
Schemas: simpleTestSchemas(),
}
g, diags := b.Build(addrs.RootModuleInstance)
@ -344,7 +340,6 @@ func TestApplyGraphBuilder_destroyCount(t *testing.T) {
Config: testModule(t, "graph-builder-apply-count"),
Changes: changes,
Plugins: simpleMockPluginLibrary(),
Schemas: simpleTestSchemas(),
State: state,
}
@ -407,7 +402,6 @@ func TestApplyGraphBuilder_moduleDestroy(t *testing.T) {
Config: testModule(t, "graph-builder-apply-module-destroy"),
Changes: changes,
Plugins: simpleMockPluginLibrary(),
Schemas: simpleTestSchemas(),
State: state,
}
@ -445,7 +439,6 @@ func TestApplyGraphBuilder_targetModule(t *testing.T) {
Config: testModule(t, "graph-builder-apply-target-module"),
Changes: changes,
Plugins: simpleMockPluginLibrary(),
Schemas: simpleTestSchemas(),
Targets: []addrs.Targetable{
addrs.RootModuleInstance.Child("child2", addrs.NoKey),
},
@ -542,7 +535,6 @@ func TestApplyGraphBuilder_updateFromOrphan(t *testing.T) {
Config: testModule(t, "graph-builder-apply-orphan-update"),
Changes: changes,
Plugins: simpleMockPluginLibrary(),
Schemas: schemas,
State: state,
}
@ -643,7 +635,6 @@ func TestApplyGraphBuilder_updateFromCBDOrphan(t *testing.T) {
Config: testModule(t, "graph-builder-apply-orphan-update"),
Changes: changes,
Plugins: simpleMockPluginLibrary(),
Schemas: schemas,
State: state,
}
@ -694,7 +685,6 @@ func TestApplyGraphBuilder_orphanedWithProvider(t *testing.T) {
Config: testModule(t, "graph-builder-orphan-alias"),
Changes: changes,
Plugins: simpleMockPluginLibrary(),
Schemas: simpleTestSchemas(),
State: state,
}

View File

@ -27,10 +27,6 @@ type DestroyPlanGraphBuilder struct {
// provisioners) available for use.
Plugins *contextPlugins
// Schemas is the repository of schemas we will draw from to analyse
// the configuration.
Schemas *Schemas
// Targets are resources to target
Targets []addrs.Targetable
@ -101,7 +97,6 @@ func (b *DestroyPlanGraphBuilder) Steps() []GraphTransformer {
&DestroyEdgeTransformer{
Config: b.Config,
State: b.State,
Schemas: b.Schemas,
},
&TargetsTransformer{Targets: b.Targets},

View File

@ -33,10 +33,6 @@ type EvalGraphBuilder struct {
// Plugins is a library of plug-in components (providers and
// provisioners) available for use.
Plugins *contextPlugins
// Schemas is the repository of schemas we will draw from to analyse
// the configuration.
Schemas *Schemas
}
// See GraphBuilder
@ -79,7 +75,7 @@ func (b *EvalGraphBuilder) Steps() []GraphTransformer {
// Must attach schemas before ReferenceTransformer so that we can
// analyze the configuration to find references.
&AttachSchemaTransformer{Schemas: b.Schemas, Config: b.Config},
&AttachSchemaTransformer{Plugins: b.Plugins, Config: b.Config},
// Create expansion nodes for all of the module calls. This must
// come after all other transformers that create nodes representing

View File

@ -20,10 +20,6 @@ type ImportGraphBuilder struct {
// Plugins is a library of plug-in components (providers and
// provisioners) available for use.
Plugins *contextPlugins
// Schemas is the repository of schemas we will draw from to analyse
// the configuration.
Schemas *Schemas
}
// Build builds the graph according to the steps returned by Steps.
@ -72,7 +68,7 @@ func (b *ImportGraphBuilder) Steps() []GraphTransformer {
// Must attach schemas before ReferenceTransformer so that we can
// analyze the configuration to find references.
&AttachSchemaTransformer{Schemas: b.Schemas, Config: b.Config},
&AttachSchemaTransformer{Plugins: b.Plugins, Config: b.Config},
// Create expansion nodes for all of the module calls. This must
// come after all other transformers that create nodes representing

View File

@ -32,10 +32,6 @@ type PlanGraphBuilder struct {
// provisioners) available for use.
Plugins *contextPlugins
// Schemas is the repository of schemas we will draw from to analyse
// the configuration.
Schemas *Schemas
// Targets are resources to target
Targets []addrs.Targetable
@ -137,7 +133,7 @@ func (b *PlanGraphBuilder) Steps() []GraphTransformer {
// Must attach schemas before ReferenceTransformer so that we can
// analyze the configuration to find references.
&AttachSchemaTransformer{Schemas: b.Schemas, Config: b.Config},
&AttachSchemaTransformer{Plugins: b.Plugins, Config: b.Config},
// Create expansion nodes for all of the module calls. This must
// come after all other transformers that create nodes representing

View File

@ -34,12 +34,6 @@ func TestPlanGraphBuilder(t *testing.T) {
b := &PlanGraphBuilder{
Config: testModule(t, "graph-builder-plan-basic"),
Plugins: plugins,
Schemas: &Schemas{
Providers: map[addrs.Provider]*ProviderSchema{
addrs.NewDefaultProvider("aws"): awsProvider.ProviderSchema(),
addrs.NewDefaultProvider("openstack"): openstackProvider.ProviderSchema(),
},
},
}
g, err := b.Build(addrs.RootModuleInstance)
@ -82,11 +76,6 @@ func TestPlanGraphBuilder_dynamicBlock(t *testing.T) {
b := &PlanGraphBuilder{
Config: testModule(t, "graph-builder-plan-dynblock"),
Plugins: plugins,
Schemas: &Schemas{
Providers: map[addrs.Provider]*ProviderSchema{
addrs.NewDefaultProvider("test"): provider.ProviderSchema(),
},
},
}
g, err := b.Build(addrs.RootModuleInstance)
@ -145,11 +134,6 @@ func TestPlanGraphBuilder_attrAsBlocks(t *testing.T) {
b := &PlanGraphBuilder{
Config: testModule(t, "graph-builder-plan-attr-as-blocks"),
Plugins: plugins,
Schemas: &Schemas{
Providers: map[addrs.Provider]*ProviderSchema{
addrs.NewDefaultProvider("test"): provider.ProviderSchema(),
},
},
}
g, err := b.Build(addrs.RootModuleInstance)
@ -190,7 +174,6 @@ func TestPlanGraphBuilder_targetModule(t *testing.T) {
b := &PlanGraphBuilder{
Config: testModule(t, "graph-builder-plan-target-module-provider"),
Plugins: simpleMockPluginLibrary(),
Schemas: simpleTestSchemas(),
Targets: []addrs.Targetable{
addrs.RootModuleInstance.Child("child2", addrs.NoKey),
},
@ -217,11 +200,6 @@ func TestPlanGraphBuilder_forEach(t *testing.T) {
b := &PlanGraphBuilder{
Config: testModule(t, "plan-for-each"),
Plugins: plugins,
Schemas: &Schemas{
Providers: map[addrs.Provider]*ProviderSchema{
addrs.NewDefaultProvider("aws"): awsProvider.ProviderSchema(),
},
},
}
g, err := b.Build(addrs.RootModuleInstance)

View File

@ -3,6 +3,7 @@ package terraform
import (
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/configs/configschema"
"github.com/hashicorp/terraform/internal/providers"
)
func simpleTestSchemas() *Schemas {
@ -18,3 +19,47 @@ func simpleTestSchemas() *Schemas {
},
}
}
// schemaOnlyProvidersForTesting is a testing helper that constructs a
// plugin library that contains a set of providers that only know how to
// return schema, and will exhibit undefined behavior if used for any other
// purpose.
//
// The intended use for this is in testing components that use schemas to
// drive other behavior, such as reference analysis during graph construction,
// but that don't actually need to interact with providers otherwise.
func schemaOnlyProvidersForTesting(schemas map[addrs.Provider]*ProviderSchema) *contextPlugins {
factories := make(map[addrs.Provider]providers.Factory, len(schemas))
for providerAddr, schema := range schemas {
resp := &providers.GetProviderSchemaResponse{
Provider: providers.Schema{
Block: schema.Provider,
},
ResourceTypes: make(map[string]providers.Schema),
DataSources: make(map[string]providers.Schema),
}
for t, tSchema := range schema.ResourceTypes {
resp.ResourceTypes[t] = providers.Schema{
Block: tSchema,
Version: int64(schema.ResourceTypeSchemaVersions[t]),
}
}
for t, tSchema := range schema.DataSources {
resp.DataSources[t] = providers.Schema{
Block: tSchema,
}
}
provider := &MockProvider{
GetProviderSchemaResponse: resp,
}
factories[providerAddr] = func() (providers.Interface, error) {
return provider, nil
}
}
return newContextPlugins(factories, nil)
}

View File

@ -43,15 +43,15 @@ type GraphNodeAttachProvisionerSchema interface {
// GraphNodeAttachProvisionerSchema, looks up the needed schemas for each
// and then passes them to a method implemented by the node.
type AttachSchemaTransformer struct {
Schemas *Schemas
Plugins *contextPlugins
Config *configs.Config
}
func (t *AttachSchemaTransformer) Transform(g *Graph) error {
if t.Schemas == nil {
if t.Plugins == nil {
// Should never happen with a reasonable caller, but we'll return a
// proper error here anyway so that we'll fail gracefully.
return fmt.Errorf("AttachSchemaTransformer used with nil Schemas")
return fmt.Errorf("AttachSchemaTransformer used with nil Plugins")
}
for _, v := range g.Vertices() {
@ -62,7 +62,10 @@ func (t *AttachSchemaTransformer) Transform(g *Graph) error {
typeName := addr.Resource.Type
providerFqn := tv.Provider()
schema, version := t.Schemas.ResourceTypeConfig(providerFqn, mode, typeName)
schema, version, err := t.Plugins.ResourceTypeSchema(providerFqn, mode, typeName)
if err != nil {
return fmt.Errorf("failed to read schema for %s in %s: %s", addr, providerFqn, err)
}
if schema == nil {
log.Printf("[ERROR] AttachSchemaTransformer: No resource schema available for %s", addr)
continue
@ -73,8 +76,10 @@ func (t *AttachSchemaTransformer) Transform(g *Graph) error {
if tv, ok := v.(GraphNodeAttachProviderConfigSchema); ok {
providerAddr := tv.ProviderAddr()
schema := t.Schemas.ProviderConfig(providerAddr.Provider)
schema, err := t.Plugins.ProviderConfigSchema(providerAddr.Provider)
if err != nil {
return fmt.Errorf("failed to read provider configuration schema for %s: %s", providerAddr.Provider, err)
}
if schema == nil {
log.Printf("[ERROR] AttachSchemaTransformer: No provider config schema available for %s", providerAddr)
continue
@ -86,7 +91,10 @@ func (t *AttachSchemaTransformer) Transform(g *Graph) error {
if tv, ok := v.(GraphNodeAttachProvisionerSchema); ok {
names := tv.ProvisionedBy()
for _, name := range names {
schema := t.Schemas.ProvisionerConfig(name)
schema, err := t.Plugins.ProvisionerSchema(name)
if err != nil {
return fmt.Errorf("failed to read provisioner configuration schema for %q: %s", name, err)
}
if schema == nil {
log.Printf("[ERROR] AttachSchemaTransformer: No schema available for provisioner %q on %q", name, dag.VertexName(v))
continue

View File

@ -115,11 +115,6 @@ type CBDEdgeTransformer struct {
// any way possible. Either can be nil if not availabile.
Config *configs.Config
State *states.State
// If configuration is present then Schemas is required in order to
// obtain schema information from providers and provisioners so we can
// properly resolve implicit dependencies.
Schemas *Schemas
}
func (t *CBDEdgeTransformer) Transform(g *Graph) error {

View File

@ -17,7 +17,6 @@ func cbdTestGraph(t *testing.T, mod string, changes *plans.Changes, state *state
Config: module,
Changes: changes,
Plugins: simpleMockPluginLibrary(),
Schemas: simpleTestSchemas(),
State: state,
}
g, err := (&BasicGraphBuilder{

View File

@ -45,11 +45,6 @@ type DestroyEdgeTransformer struct {
// to determine what a destroy node depends on. Any of these can be nil.
Config *configs.Config
State *states.State
// If configuration is present then Schemas is required in order to
// obtain schema information from providers and provisioners in order
// to properly resolve implicit dependencies.
Schemas *Schemas
}
func (t *DestroyEdgeTransformer) Transform(g *Graph) error {

View File

@ -39,7 +39,6 @@ func TestDestroyEdgeTransformer_basic(t *testing.T) {
tf := &DestroyEdgeTransformer{
Config: testModule(t, "transform-destroy-edge-basic"),
Schemas: simpleTestSchemas(),
}
if err := tf.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
@ -96,7 +95,6 @@ func TestDestroyEdgeTransformer_multi(t *testing.T) {
tf := &DestroyEdgeTransformer{
Config: testModule(t, "transform-destroy-edge-multi"),
Schemas: simpleTestSchemas(),
}
if err := tf.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
@ -114,7 +112,6 @@ func TestDestroyEdgeTransformer_selfRef(t *testing.T) {
g.Add(testDestroyNode("test_object.A"))
tf := &DestroyEdgeTransformer{
Config: testModule(t, "transform-destroy-edge-self-ref"),
Schemas: simpleTestSchemas(),
}
if err := tf.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
@ -158,7 +155,6 @@ func TestDestroyEdgeTransformer_module(t *testing.T) {
tf := &DestroyEdgeTransformer{
Config: testModule(t, "transform-destroy-edge-module"),
Schemas: simpleTestSchemas(),
}
if err := tf.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
@ -220,7 +216,6 @@ func TestDestroyEdgeTransformer_moduleOnly(t *testing.T) {
tf := &DestroyEdgeTransformer{
Config: testModule(t, "transform-destroy-edge-module-only"),
Schemas: simpleTestSchemas(),
}
if err := tf.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
@ -298,7 +293,6 @@ resource "test_instance" "a" {
})
tf := &DestroyEdgeTransformer{
Config: m,
Schemas: simpleTestSchemas(),
}
if err := tf.Transform(&g); err != nil {
t.Fatalf("err: %s", err)

View File

@ -30,11 +30,10 @@ func TestTransitiveReductionTransformer(t *testing.T) {
{
transform := &AttachSchemaTransformer{
Schemas: &Schemas{
Providers: map[addrs.Provider]*ProviderSchema{
Plugins: schemaOnlyProvidersForTesting(map[addrs.Provider]*ProviderSchema{
addrs.NewDefaultProvider("aws"): {
ResourceTypes: map[string]*configschema.Block{
"aws_instance": &configschema.Block{
"aws_instance": {
Attributes: map[string]*configschema.Attribute{
"A": {
Type: cty.String,
@ -48,8 +47,7 @@ func TestTransitiveReductionTransformer(t *testing.T) {
},
},
},
},
},
}),
}
if err := transform.Transform(&g); err != nil {
t.Fatalf("err: %s", err)