From 28d406c040697bedc9c593e8a023176a0401be26 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Wed, 9 Nov 2016 09:58:52 -0500 Subject: [PATCH] Provider a marshaler for dag.Graph The dot format generation was done with a mix of code from the terraform package and the dot package. Unify the dot generation code, and it into the dag package. Use an intermediate structure to allow a dag.Graph to marshal itself directly. This structure will be ablt to marshal directly to JSON, or be translated to dot format. This was we can record more information about the graph in the debug logs, and provide a way to translate those logged structures to dot, which is convenient for viewing the graphs. --- command/graph.go | 3 +- dag/dot.go | 230 ++++++++++++++++++++++++ dag/graph.go | 15 ++ dag/marshal.go | 150 ++++++++++++++++ terraform/graph_config_node_module.go | 2 +- terraform/graph_config_node_provider.go | 2 +- terraform/graph_config_node_resource.go | 2 +- terraform/graph_debug.go | 12 +- terraform/graph_dot.go | 22 +-- terraform/graph_dot_test.go | 87 +++++---- terraform/transform_provider.go | 4 +- terraform/transform_provider_old.go | 2 +- 12 files changed, 473 insertions(+), 58 deletions(-) create mode 100644 dag/dot.go create mode 100644 dag/marshal.go diff --git a/command/graph.go b/command/graph.go index 6c7bfae04..ac59429a4 100644 --- a/command/graph.go +++ b/command/graph.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/hashicorp/terraform/dag" "github.com/hashicorp/terraform/terraform" ) @@ -67,7 +68,7 @@ func (c *GraphCommand) Run(args []string) int { return 1 } - graphStr, err := terraform.GraphDot(g, &terraform.GraphDotOpts{ + graphStr, err := terraform.GraphDot(g, &dag.DotOpts{ DrawCycles: drawCycles, MaxDepth: moduleDepth, Verbose: verbose, diff --git a/dag/dot.go b/dag/dot.go new file mode 100644 index 000000000..f52cfafba --- /dev/null +++ b/dag/dot.go @@ -0,0 +1,230 @@ +package dag + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +// DotOpts are the options for generating a dot formatted Graph. +type DotOpts struct { + // Allows some nodes to decide to only show themselves when the user has + // requested the "verbose" graph. + Verbose bool + + // Highlight Cycles + DrawCycles bool + + // How many levels to expand modules as we draw + MaxDepth int + + // use this to keep the cluster_ naming convention from the previous dot writer + cluster bool +} + +// Returns the DOT representation of this Graph. +func (g *marshalGraph) Dot(opts *DotOpts) []byte { + if opts == nil { + opts = &DotOpts{ + DrawCycles: true, + MaxDepth: -1, + Verbose: true, + } + } + + var w indentWriter + w.WriteString("digraph {\n") + w.Indent() + + // some dot defaults + w.WriteString(`compound = "true"` + "\n") + w.WriteString(`newrank = "true"` + "\n") + + // the top level graph is written as the first subgraph + w.WriteString(`subgraph "root" {` + "\n") + g.writeBody(opts, &w) + + // cluster isn't really used other than for naming purposes in some graphs + opts.cluster = opts.MaxDepth != 0 + maxDepth := opts.MaxDepth + if maxDepth == 0 { + maxDepth = -1 + } + + for _, s := range g.Subgraphs { + g.writeSubgraph(s, opts, maxDepth, &w) + } + + w.Unindent() + w.WriteString("}\n") + return w.Bytes() +} + +func (v *marshalVertex) dot(g *marshalGraph) []byte { + var buf bytes.Buffer + graphName := g.Name + if graphName == "" { + graphName = "root" + } + buf.WriteString(fmt.Sprintf(`"[%s] %s"`, graphName, v.Name)) + writeAttrs(&buf, v.Attrs) + buf.WriteByte('\n') + + return buf.Bytes() +} + +func (e *marshalEdge) dot(g *marshalGraph) string { + var buf bytes.Buffer + graphName := g.Name + if graphName == "" { + graphName = "root" + } + + sourceName := g.vertexByID(e.Source).Name + targetName := g.vertexByID(e.Target).Name + s := fmt.Sprintf(`"[%s] %s" -> "[%s] %s"`, graphName, sourceName, graphName, targetName) + buf.WriteString(s) + writeAttrs(&buf, e.Attrs) + + return buf.String() +} + +func cycleDot(e *marshalEdge, g *marshalGraph) string { + return e.dot(g) + ` [color = "red", penwidth = "2.0"]` +} + +// Write the subgraph body. The is recursive, and the depth argument is used to +// record the current depth of iteration. +func (g *marshalGraph) writeSubgraph(sg *marshalGraph, opts *DotOpts, depth int, w *indentWriter) { + if depth == 0 { + return + } + depth-- + + name := sg.Name + if opts.cluster { + // we prefix with cluster_ to match the old dot output + name = "cluster_" + name + sg.Attrs["label"] = sg.Name + } + w.WriteString(fmt.Sprintf("subgraph %q {\n", name)) + sg.writeBody(opts, w) + + for _, sg := range sg.Subgraphs { + g.writeSubgraph(sg, opts, depth, w) + } +} + +func (g *marshalGraph) writeBody(opts *DotOpts, w *indentWriter) { + w.Indent() + + for _, as := range attrStrings(g.Attrs) { + w.WriteString(as + "\n") + } + + for _, v := range g.Vertices { + w.Write(v.dot(g)) + } + + var dotEdges []string + + if opts.DrawCycles { + for _, c := range g.Cycles { + if len(c) < 2 { + continue + } + + for i, j := 0, 1; i < len(c); i, j = i+1, j+1 { + if j >= len(c) { + j = 0 + } + src := c[i] + tgt := c[j] + e := &marshalEdge{ + Name: fmt.Sprintf("%s|%s", src.Name, tgt.Name), + Source: src.ID, + Target: tgt.ID, + Attrs: make(map[string]string), + } + + dotEdges = append(dotEdges, cycleDot(e, g)) + src = tgt + } + } + } + + for _, e := range g.Edges { + dotEdges = append(dotEdges, e.dot(g)) + } + + // srot these again to match the old output + sort.Strings(dotEdges) + + for _, e := range dotEdges { + w.WriteString(e + "\n") + } + + w.Unindent() + w.WriteString("}\n") +} + +func writeAttrs(buf *bytes.Buffer, attrs map[string]string) { + if len(attrs) > 0 { + buf.WriteString(" [") + buf.WriteString(strings.Join(attrStrings(attrs), ", ")) + buf.WriteString("]") + } +} + +func attrStrings(attrs map[string]string) []string { + strings := make([]string, 0, len(attrs)) + for k, v := range attrs { + strings = append(strings, fmt.Sprintf("%s = %q", k, v)) + } + sort.Strings(strings) + return strings +} + +// Provide a bytes.Buffer like structure, which will indent when starting a +// newline. +type indentWriter struct { + bytes.Buffer + level int +} + +func (w *indentWriter) indent() { + newline := []byte("\n") + if !bytes.HasSuffix(w.Bytes(), newline) { + return + } + for i := 0; i < w.level; i++ { + w.Buffer.WriteString("\t") + } +} + +// Indent increases indentation by 1 +func (w *indentWriter) Indent() { w.level++ } + +// Unindent decreases indentation by 1 +func (w *indentWriter) Unindent() { w.level-- } + +// the following methods intercecpt the byte.Buffer writes and insert the +// indentation when starting a new line. +func (w *indentWriter) Write(b []byte) (int, error) { + w.indent() + return w.Buffer.Write(b) +} + +func (w *indentWriter) WriteString(s string) (int, error) { + w.indent() + return w.Buffer.WriteString(s) +} +func (w *indentWriter) WriteByte(b byte) error { + w.indent() + return w.Buffer.WriteByte(b) +} +func (w *indentWriter) WriteRune(r rune) (int, error) { + w.indent() + return w.Buffer.WriteRune(r) +} diff --git a/dag/graph.go b/dag/graph.go index 75bc5dbe5..e713ff3ef 100644 --- a/dag/graph.go +++ b/dag/graph.go @@ -2,6 +2,7 @@ package dag import ( "bytes" + "encoding/json" "fmt" "sort" "sync" @@ -284,6 +285,20 @@ func (g *Graph) String() string { return buf.String() } +func (g *Graph) Marshal() ([]byte, error) { + dg := newMarshalGraph("", g) + return json.MarshalIndent(dg, "", " ") +} + +func (g *Graph) Dot(opts *DotOpts) []byte { + return newMarshalGraph("", g).Dot(opts) +} + +func (g *Graph) MarshalJSON() ([]byte, error) { + dg := newMarshalGraph("", g) + return json.MarshalIndent(dg, "", " ") +} + func (g *Graph) init() { g.vertices = new(Set) g.edges = new(Set) diff --git a/dag/marshal.go b/dag/marshal.go new file mode 100644 index 000000000..7b0f1e8e3 --- /dev/null +++ b/dag/marshal.go @@ -0,0 +1,150 @@ +package dag + +import ( + "fmt" + "reflect" + "sort" + "strconv" +) + +// the marshal* structs are for serialization of the graph data. +type marshalGraph struct { + ID string `json:",omitempty"` + Name string `json:",omitempty"` + Attrs map[string]string `json:",omitempty"` + Vertices []*marshalVertex `json:",omitempty"` + Edges []*marshalEdge `json:",omitempty"` + Subgraphs []*marshalGraph `json:",omitempty"` + Cycles [][]*marshalVertex `json:",omitempty"` +} + +func (g *marshalGraph) vertexByID(id string) *marshalVertex { + for _, v := range g.Vertices { + if id == v.ID { + return v + } + } + return nil +} + +type marshalVertex struct { + ID string + Name string `json:",omitempty"` + Attrs map[string]string `json:",omitempty"` +} + +type vertices []*marshalVertex + +func (v vertices) Less(i, j int) bool { return v[i].Name < v[j].Name } +func (v vertices) Len() int { return len(v) } +func (v vertices) Swap(i, j int) { v[i], v[j] = v[j], v[i] } + +type marshalEdge struct { + Name string + Source string + Target string + Attrs map[string]string `json:",omitempty"` +} + +type edges []*marshalEdge + +func (e edges) Less(i, j int) bool { return e[i].Name < e[j].Name } +func (e edges) Len() int { return len(e) } +func (e edges) Swap(i, j int) { e[i], e[j] = e[j], e[i] } + +func newMarshalGraph(name string, g *Graph) *marshalGraph { + dg := &marshalGraph{ + Name: name, + Attrs: make(map[string]string), + } + + for _, v := range g.Vertices() { + id := marshalVertexID(v) + if sg, ok := marshalSubgraph(v); ok { + + sdg := newMarshalGraph(VertexName(v), sg) + sdg.ID = id + dg.Subgraphs = append(dg.Subgraphs, sdg) + } + + dv := &marshalVertex{ + ID: id, + Name: VertexName(v), + Attrs: make(map[string]string), + } + + dg.Vertices = append(dg.Vertices, dv) + } + + sort.Sort(vertices(dg.Vertices)) + + for _, e := range g.Edges() { + de := &marshalEdge{ + Name: fmt.Sprintf("%s|%s", VertexName(e.Source()), VertexName(e.Target())), + Source: marshalVertexID(e.Source()), + Target: marshalVertexID(e.Target()), + Attrs: make(map[string]string), + } + dg.Edges = append(dg.Edges, de) + } + + sort.Sort(edges(dg.Edges)) + + for _, c := range (&AcyclicGraph{*g}).Cycles() { + var cycle []*marshalVertex + for _, v := range c { + dv := &marshalVertex{ + ID: marshalVertexID(v), + Name: VertexName(v), + Attrs: make(map[string]string), + } + + cycle = append(cycle, dv) + } + dg.Cycles = append(dg.Cycles, cycle) + } + + return dg +} + +// Attempt to return a unique ID for any vertex. +func marshalVertexID(v Vertex) string { + val := reflect.ValueOf(v) + switch val.Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Slice, reflect.UnsafePointer: + return strconv.Itoa(int(val.Pointer())) + case reflect.Interface: + return strconv.Itoa(int(val.InterfaceData()[1])) + } + + if v, ok := v.(Hashable); ok { + h := v.Hashcode() + if h, ok := h.(string); ok { + return h + } + } + + // we could try harder by attempting to read the arbitrary value from the + // interface, but we shouldn't get here from terraform right now. + panic("unhashable value in graph") +} + +func debugSubgraph(v Vertex) (*Graph, bool) { + val := reflect.ValueOf(v) + m, ok := val.Type().MethodByName("Subgraph") + if !ok { + return nil, false + } + + if m.Type.NumOut() != 1 { + return nil, false + } + + // can't check for the subgraph type, because we can't import terraform, so + // we assume this is the correct method. + // TODO: create a dag interface type that we can satisfy + + sg := val.MethodByName("Subgraph").Call(nil)[0] + ag := sg.Elem().FieldByName("AcyclicGraph").Interface().(AcyclicGraph) + return &ag.Graph, true +} diff --git a/terraform/graph_config_node_module.go b/terraform/graph_config_node_module.go index 8da8fb6cd..4396c2e65 100644 --- a/terraform/graph_config_node_module.go +++ b/terraform/graph_config_node_module.go @@ -129,7 +129,7 @@ func (n *graphNodeModuleExpanded) DependentOn() []string { } // GraphNodeDotter impl. -func (n *graphNodeModuleExpanded) DotNode(name string, opts *GraphDotOpts) *dot.Node { +func (n *graphNodeModuleExpanded) DotNode(name string, opts *dag.DotOpts) *dot.Node { return dot.NewNode(name, map[string]string{ "label": dag.VertexName(n.Original), "shape": "component", diff --git a/terraform/graph_config_node_provider.go b/terraform/graph_config_node_provider.go index c0e15eff3..f42d1430c 100644 --- a/terraform/graph_config_node_provider.go +++ b/terraform/graph_config_node_provider.go @@ -59,7 +59,7 @@ func (n *GraphNodeConfigProvider) ProviderConfig() *config.RawConfig { } // GraphNodeDotter impl. -func (n *GraphNodeConfigProvider) DotNode(name string, opts *GraphDotOpts) *dot.Node { +func (n *GraphNodeConfigProvider) DotNode(name string, opts *dag.DotOpts) *dot.Node { return dot.NewNode(name, map[string]string{ "label": n.Name(), "shape": "diamond", diff --git a/terraform/graph_config_node_resource.go b/terraform/graph_config_node_resource.go index b160f6ddd..27654d2bb 100644 --- a/terraform/graph_config_node_resource.go +++ b/terraform/graph_config_node_resource.go @@ -128,7 +128,7 @@ func (n *GraphNodeConfigResource) Name() string { } // GraphNodeDotter impl. -func (n *GraphNodeConfigResource) DotNode(name string, opts *GraphDotOpts) *dot.Node { +func (n *GraphNodeConfigResource) DotNode(name string, opts *dag.DotOpts) *dot.Node { if n.Destroy && !opts.Verbose { return nil } diff --git a/terraform/graph_debug.go b/terraform/graph_debug.go index 6b9e5f096..80c84e780 100644 --- a/terraform/graph_debug.go +++ b/terraform/graph_debug.go @@ -28,14 +28,14 @@ type DebugGraph struct { buf bytes.Buffer Dot *dot.Graph - dotOpts *GraphDotOpts + dotOpts *dag.DotOpts } // DebugGraph holds a dot representation of the Terraform graph, and can be // written out to the DebugInfo log with DebugInfo.WriteGraph. A DebugGraph can // log data to it's internal buffer via the Printf and Write methods, which // will be also be written out to the DebugInfo archive. -func NewDebugGraph(name string, g *Graph, opts *GraphDotOpts) (*DebugGraph, error) { +func NewDebugGraph(name string, g *Graph, opts *dag.DotOpts) (*DebugGraph, error) { dg := &DebugGraph{ Name: name, dotOpts: opts, @@ -142,7 +142,7 @@ func (dg *DebugGraph) build(g *Graph) error { dg.Dot.Directed = true if dg.dotOpts == nil { - dg.dotOpts = &GraphDotOpts{ + dg.dotOpts = &dag.DotOpts{ DrawCycles: true, MaxDepth: -1, Verbose: true, @@ -182,6 +182,12 @@ func (dg *DebugGraph) buildSubgraph(modName string, g *Graph, modDepth int) erro toDraw := make([]dag.Vertex, 0, len(g.Vertices())) subgraphVertices := make(map[dag.Vertex]*Graph) + for _, v := range g.Vertices() { + if sn, ok := v.(GraphNodeSubgraph); ok { + subgraphVertices[v] = sn.Subgraph() + } + } + walk := func(v dag.Vertex, depth int) error { // We only care about nodes that yield non-empty Dot strings. if dn, ok := v.(GraphNodeDotter); !ok { diff --git a/terraform/graph_dot.go b/terraform/graph_dot.go index c7a953796..8eeec7592 100644 --- a/terraform/graph_dot.go +++ b/terraform/graph_dot.go @@ -1,6 +1,9 @@ package terraform -import "github.com/hashicorp/terraform/dot" +import ( + "github.com/hashicorp/terraform/dag" + "github.com/hashicorp/terraform/dot" +) // GraphNodeDotter can be implemented by a node to cause it to be included // in the dot graph. The Dot method will be called which is expected to @@ -10,25 +13,12 @@ type GraphNodeDotter interface { // The first parameter is the title of the node. // The second parameter includes user-specified options that affect the dot // graph. See GraphDotOpts below for details. - DotNode(string, *GraphDotOpts) *dot.Node -} - -// GraphDotOpts are the options for generating a dot formatted Graph. -type GraphDotOpts struct { - // Allows some nodes to decide to only show themselves when the user has - // requested the "verbose" graph. - Verbose bool - - // Highlight Cycles - DrawCycles bool - - // How many levels to expand modules as we draw - MaxDepth int + DotNode(string, *dag.DotOpts) *dot.Node } // GraphDot returns the dot formatting of a visual representation of // the given Terraform graph. -func GraphDot(g *Graph, opts *GraphDotOpts) (string, error) { +func GraphDot(g *Graph, opts *dag.DotOpts) (string, error) { dg, err := NewDebugGraph("root", g, opts) if err != nil { return "", err diff --git a/terraform/graph_dot_test.go b/terraform/graph_dot_test.go index ecef1984d..cc4376e46 100644 --- a/terraform/graph_dot_test.go +++ b/terraform/graph_dot_test.go @@ -4,21 +4,31 @@ import ( "strings" "testing" + "github.com/hashicorp/terraform/dag" "github.com/hashicorp/terraform/dot" ) func TestGraphDot(t *testing.T) { - cases := map[string]struct { + cases := []struct { + Name string Graph testGraphFunc - Opts GraphDotOpts + Opts dag.DotOpts Expect string Error string }{ - "empty": { + { + Name: "empty", Graph: func() *Graph { return &Graph{} }, - Error: "No DOT origin nodes found", + Expect: ` +digraph { + compound = "true" + newrank = "true" + subgraph "root" { + } +}`, }, - "three-level": { + { + Name: "three-level", Graph: func() *Graph { var g Graph root := &testDrawableOrigin{"root"} @@ -61,8 +71,10 @@ digraph { } `, }, - "cycle": { - Opts: GraphDotOpts{ + + { + Name: "cycle", + Opts: dag.DotOpts{ DrawCycles: true, }, Graph: func() *Graph { @@ -108,8 +120,10 @@ digraph { } `, }, - "subgraphs, no depth restriction": { - Opts: GraphDotOpts{ + + { + Name: "subgraphs, no depth restriction", + Opts: dag.DotOpts{ MaxDepth: -1, }, Graph: func() *Graph { @@ -159,8 +173,10 @@ digraph { } `, }, - "subgraphs, with depth restriction": { - Opts: GraphDotOpts{ + + { + Name: "subgraphs, with depth restriction", + Opts: dag.DotOpts{ MaxDepth: 1, }, Graph: func() *Graph { @@ -208,25 +224,32 @@ digraph { }, } - for tn, tc := range cases { - actual, err := GraphDot(tc.Graph(), &tc.Opts) - if err == nil && tc.Error != "" { - t.Fatalf("%s: expected err: %s, got none", tn, tc.Error) - } - if err != nil && tc.Error == "" { - t.Fatalf("%s: unexpected err: %s", tn, err) - } - if err != nil && tc.Error != "" { - if !strings.Contains(err.Error(), tc.Error) { - t.Fatalf("%s: expected err: %s\nto contain: %s", tn, err, tc.Error) - } - continue - } + for _, tc := range cases { + tn := tc.Name + t.Run(tn, func(t *testing.T) { + g := tc.Graph() + var err error + //actual, err := GraphDot(g, &tc.Opts) + actual := string(g.Dot(&tc.Opts)) - expected := strings.TrimSpace(tc.Expect) + "\n" - if actual != expected { - t.Fatalf("%s:\n\nexpected:\n%s\n\ngot:\n%s", tn, expected, actual) - } + if err == nil && tc.Error != "" { + t.Fatalf("%s: expected err: %s, got none", tn, tc.Error) + } + if err != nil && tc.Error == "" { + t.Fatalf("%s: unexpected err: %s", tn, err) + } + if err != nil && tc.Error != "" { + if !strings.Contains(err.Error(), tc.Error) { + t.Fatalf("%s: expected err: %s\nto contain: %s", tn, err, tc.Error) + } + return + } + + expected := strings.TrimSpace(tc.Expect) + "\n" + if actual != expected { + t.Fatalf("%s:\n\nexpected:\n%s\n\ngot:\n%s", tn, expected, actual) + } + }) } } @@ -240,7 +263,7 @@ type testDrawable struct { func (node *testDrawable) Name() string { return node.VertexName } -func (node *testDrawable) DotNode(n string, opts *GraphDotOpts) *dot.Node { +func (node *testDrawable) DotNode(n string, opts *dag.DotOpts) *dot.Node { return dot.NewNode(n, map[string]string{}) } func (node *testDrawable) DependableName() []string { @@ -257,7 +280,7 @@ type testDrawableOrigin struct { func (node *testDrawableOrigin) Name() string { return node.VertexName } -func (node *testDrawableOrigin) DotNode(n string, opts *GraphDotOpts) *dot.Node { +func (node *testDrawableOrigin) DotNode(n string, opts *dag.DotOpts) *dot.Node { return dot.NewNode(n, map[string]string{}) } func (node *testDrawableOrigin) DotOrigin() bool { @@ -279,7 +302,7 @@ func (node *testDrawableSubgraph) Name() string { func (node *testDrawableSubgraph) Subgraph() *Graph { return node.SubgraphMock } -func (node *testDrawableSubgraph) DotNode(n string, opts *GraphDotOpts) *dot.Node { +func (node *testDrawableSubgraph) DotNode(n string, opts *dag.DotOpts) *dot.Node { return dot.NewNode(n, map[string]string{}) } func (node *testDrawableSubgraph) DependentOn() []string { diff --git a/terraform/transform_provider.go b/terraform/transform_provider.go index 606b81dcd..f46af247c 100644 --- a/terraform/transform_provider.go +++ b/terraform/transform_provider.go @@ -355,7 +355,7 @@ func (n *graphNodeCloseProvider) CloseProviderName() string { } // GraphNodeDotter impl. -func (n *graphNodeCloseProvider) DotNode(name string, opts *GraphDotOpts) *dot.Node { +func (n *graphNodeCloseProvider) DotNode(name string, opts *dag.DotOpts) *dot.Node { if !opts.Verbose { return nil } @@ -393,7 +393,7 @@ func (n *graphNodeProvider) ProviderConfig() *config.RawConfig { } // GraphNodeDotter impl. -func (n *graphNodeProvider) DotNode(name string, opts *GraphDotOpts) *dot.Node { +func (n *graphNodeProvider) DotNode(name string, opts *dag.DotOpts) *dot.Node { return dot.NewNode(name, map[string]string{ "label": n.Name(), "shape": "diamond", diff --git a/terraform/transform_provider_old.go b/terraform/transform_provider_old.go index eb533f230..1e49294b7 100644 --- a/terraform/transform_provider_old.go +++ b/terraform/transform_provider_old.go @@ -102,7 +102,7 @@ func (n *graphNodeDisabledProvider) Name() string { } // GraphNodeDotter impl. -func (n *graphNodeDisabledProvider) DotNode(name string, opts *GraphDotOpts) *dot.Node { +func (n *graphNodeDisabledProvider) DotNode(name string, opts *dag.DotOpts) *dot.Node { return dot.NewNode(name, map[string]string{ "label": n.Name(), "shape": "diamond",