374 lines
12 KiB
Go
374 lines
12 KiB
Go
package views
|
|
|
|
import (
|
|
"encoding/xml"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/hashicorp/terraform/command/arguments"
|
|
"github.com/hashicorp/terraform/command/format"
|
|
"github.com/hashicorp/terraform/internal/moduletest"
|
|
"github.com/hashicorp/terraform/internal/terminal"
|
|
"github.com/hashicorp/terraform/internal/tfdiags"
|
|
"github.com/mitchellh/colorstring"
|
|
)
|
|
|
|
// Test is the view interface for the "terraform test" command.
|
|
type Test interface {
|
|
// Results presents the given test results.
|
|
Results(map[string]*moduletest.Suite) tfdiags.Diagnostics
|
|
|
|
// Diagnostics is for reporting warnings or errors that occurred with the
|
|
// mechanics of running tests. For this command in particular, some
|
|
// errors are considered to be test failures rather than mechanism failures,
|
|
// and so those will be reported via Results rather than via Diagnostics.
|
|
Diagnostics(tfdiags.Diagnostics)
|
|
}
|
|
|
|
// NewTest returns an implementation of Test configured to respect the
|
|
// settings described in the given arguments.
|
|
func NewTest(base *View, args arguments.TestOutput) Test {
|
|
return &testHuman{
|
|
streams: base.streams,
|
|
showDiagnostics: base.Diagnostics,
|
|
colorize: base.colorize,
|
|
junitXMLFile: args.JUnitXMLFile,
|
|
}
|
|
}
|
|
|
|
type testHuman struct {
|
|
// This is the subset of functionality we need from the base view.
|
|
streams *terminal.Streams
|
|
showDiagnostics func(diags tfdiags.Diagnostics)
|
|
colorize *colorstring.Colorize
|
|
|
|
// If junitXMLFile is not empty then results will be written to
|
|
// the given file path in addition to the usual output.
|
|
junitXMLFile string
|
|
}
|
|
|
|
func (v *testHuman) Results(results map[string]*moduletest.Suite) tfdiags.Diagnostics {
|
|
var diags tfdiags.Diagnostics
|
|
|
|
// FIXME: Due to how this prototype command evolved concurrently with
|
|
// establishing the idea of command views, the handling of JUnit output
|
|
// as part of the "human" view rather than as a separate view in its
|
|
// own right is a little odd and awkward. We should refactor this
|
|
// prior to making "terraform test" a real supported command to make
|
|
// it be structured more like the other commands that use the views
|
|
// package.
|
|
|
|
v.humanResults(results)
|
|
|
|
if v.junitXMLFile != "" {
|
|
moreDiags := v.junitXMLResults(results, v.junitXMLFile)
|
|
diags = diags.Append(moreDiags)
|
|
}
|
|
|
|
return diags
|
|
}
|
|
|
|
func (v *testHuman) Diagnostics(diags tfdiags.Diagnostics) {
|
|
if len(diags) == 0 {
|
|
return
|
|
}
|
|
v.showDiagnostics(diags)
|
|
}
|
|
|
|
func (v *testHuman) humanResults(results map[string]*moduletest.Suite) {
|
|
failCount := 0
|
|
width := v.streams.Stderr.Columns()
|
|
|
|
suiteNames := make([]string, 0, len(results))
|
|
for suiteName := range results {
|
|
suiteNames = append(suiteNames, suiteName)
|
|
}
|
|
sort.Strings(suiteNames)
|
|
for _, suiteName := range suiteNames {
|
|
suite := results[suiteName]
|
|
|
|
componentNames := make([]string, 0, len(suite.Components))
|
|
for componentName := range suite.Components {
|
|
componentNames = append(componentNames, componentName)
|
|
}
|
|
for _, componentName := range componentNames {
|
|
component := suite.Components[componentName]
|
|
|
|
assertionNames := make([]string, 0, len(component.Assertions))
|
|
for assertionName := range component.Assertions {
|
|
assertionNames = append(assertionNames, assertionName)
|
|
}
|
|
sort.Strings(assertionNames)
|
|
|
|
for _, assertionName := range assertionNames {
|
|
assertion := component.Assertions[assertionName]
|
|
|
|
fullName := fmt.Sprintf("%s.%s.%s", suiteName, componentName, assertionName)
|
|
if strings.HasPrefix(componentName, "(") {
|
|
// parenthesis-prefixed components are placeholders that
|
|
// the test harness generates to represent problems that
|
|
// prevented checking any assertions at all, so we'll
|
|
// just hide them and show the suite name.
|
|
fullName = suiteName
|
|
}
|
|
headingExtra := fmt.Sprintf("%s (%s)", fullName, assertion.Description)
|
|
|
|
switch assertion.Outcome {
|
|
case moduletest.Failed:
|
|
// Failed means that the assertion was successfully
|
|
// excecuted but that the assertion condition didn't hold.
|
|
v.eprintRuleHeading("yellow", "Failed", headingExtra)
|
|
|
|
case moduletest.Error:
|
|
// Error means that the system encountered an unexpected
|
|
// error when trying to evaluate the assertion.
|
|
v.eprintRuleHeading("red", "Error", headingExtra)
|
|
|
|
default:
|
|
// We don't do anything for moduletest.Passed or
|
|
// moduletest.Skipped. Perhaps in future we'll offer a
|
|
// -verbose option to include information about those.
|
|
continue
|
|
}
|
|
failCount++
|
|
|
|
if len(assertion.Message) > 0 {
|
|
dispMsg := format.WordWrap(assertion.Message, width)
|
|
v.streams.Eprintln(dispMsg)
|
|
}
|
|
if len(assertion.Diagnostics) > 0 {
|
|
// We'll do our own writing of the diagnostics in this
|
|
// case, rather than using v.Diagnostics, because we
|
|
// specifically want all of these diagnostics to go to
|
|
// Stderr along with all of the other output we've
|
|
// generated.
|
|
for _, diag := range assertion.Diagnostics {
|
|
diagStr := format.Diagnostic(diag, nil, v.colorize, width)
|
|
v.streams.Eprint(diagStr)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if failCount > 0 {
|
|
// If we've printed at least one failure then we'll have printed at
|
|
// least one horizontal rule across the terminal, and so we'll balance
|
|
// that with another horizontal rule.
|
|
if width > 1 {
|
|
rule := strings.Repeat("─", width-1)
|
|
v.streams.Eprintln(v.colorize.Color("[dark_gray]" + rule))
|
|
}
|
|
}
|
|
|
|
if failCount == 0 {
|
|
if len(results) > 0 {
|
|
// This is not actually an error, but it's convenient if all of our
|
|
// result output goes to the same stream for when this is running in
|
|
// automation that might be gathering this output via a pipe.
|
|
v.streams.Eprint(v.colorize.Color("[bold][green]Success![reset] All of the test assertions passed.\n\n"))
|
|
} else {
|
|
v.streams.Eprint(v.colorize.Color("[bold][yellow]No tests defined.[reset] This module doesn't have any test suites to run.\n\n"))
|
|
}
|
|
}
|
|
|
|
// Try to flush any buffering that might be happening. (This isn't always
|
|
// successful, depending on what sort of fd Stderr is connected to.)
|
|
v.streams.Stderr.File.Sync()
|
|
}
|
|
|
|
func (v *testHuman) junitXMLResults(results map[string]*moduletest.Suite, filename string) tfdiags.Diagnostics {
|
|
var diags tfdiags.Diagnostics
|
|
|
|
// "JUnit XML" is a file format that has become a de-facto standard for
|
|
// test reporting tools but that is not formally specified anywhere, and
|
|
// so each producer and consumer implementation unfortunately tends to
|
|
// differ in certain ways from others.
|
|
// With that in mind, this is a best effort sort of thing aimed at being
|
|
// broadly compatible with various consumers, but it's likely that
|
|
// some consumers will present these results better than others.
|
|
// This implementation is based mainly on the pseudo-specification of the
|
|
// format curated here, based on the Jenkins parser implementation:
|
|
// https://llg.cubic.org/docs/junit/
|
|
|
|
// An "Outcome" represents one of the various XML elements allowed inside
|
|
// a testcase element to indicate the test outcome.
|
|
type Outcome struct {
|
|
Message string `xml:"message,omitempty"`
|
|
}
|
|
|
|
// TestCase represents an individual test case as part of a suite. Note
|
|
// that a JUnit XML incorporates both the "component" and "assertion"
|
|
// levels of our model: we pretend that component is a class name and
|
|
// assertion is a method name in order to match with the Java-flavored
|
|
// expectations of JUnit XML, which are hopefully close enough to get
|
|
// a test result rendering that's useful to humans.
|
|
type TestCase struct {
|
|
AssertionName string `xml:"name"`
|
|
ComponentName string `xml:"classname"`
|
|
|
|
// These fields represent the different outcomes of a TestCase. Only one
|
|
// of these should be populated in each TestCase; this awkward
|
|
// structure is just to make this play nicely with encoding/xml's
|
|
// expecatations.
|
|
Skipped *Outcome `xml:"skipped,omitempty"`
|
|
Error *Outcome `xml:"error,omitempty"`
|
|
Failure *Outcome `xml:"failure,omitempty"`
|
|
|
|
Stderr string `xml:"system-out,omitempty"`
|
|
}
|
|
|
|
// TestSuite represents an individual test suite, of potentially many
|
|
// in a JUnit XML document.
|
|
type TestSuite struct {
|
|
Name string `xml:"name"`
|
|
TotalCount int `xml:"tests"`
|
|
SkippedCount int `xml:"skipped"`
|
|
ErrorCount int `xml:"errors"`
|
|
FailureCount int `xml:"failures"`
|
|
Cases []*TestCase `xml:"testcase"`
|
|
}
|
|
|
|
// TestSuites represents the root element of the XML document.
|
|
type TestSuites struct {
|
|
XMLName struct{} `xml:"testsuites"`
|
|
ErrorCount int `xml:"errors"`
|
|
FailureCount int `xml:"failures"`
|
|
TotalCount int `xml:"tests"`
|
|
Suites []*TestSuite `xml:"testsuite"`
|
|
}
|
|
|
|
xmlSuites := TestSuites{}
|
|
suiteNames := make([]string, 0, len(results))
|
|
for suiteName := range results {
|
|
suiteNames = append(suiteNames, suiteName)
|
|
}
|
|
sort.Strings(suiteNames)
|
|
for _, suiteName := range suiteNames {
|
|
suite := results[suiteName]
|
|
|
|
xmlSuite := &TestSuite{
|
|
Name: suiteName,
|
|
}
|
|
xmlSuites.Suites = append(xmlSuites.Suites, xmlSuite)
|
|
|
|
componentNames := make([]string, 0, len(suite.Components))
|
|
for componentName := range suite.Components {
|
|
componentNames = append(componentNames, componentName)
|
|
}
|
|
for _, componentName := range componentNames {
|
|
component := suite.Components[componentName]
|
|
|
|
assertionNames := make([]string, 0, len(component.Assertions))
|
|
for assertionName := range component.Assertions {
|
|
assertionNames = append(assertionNames, assertionName)
|
|
}
|
|
sort.Strings(assertionNames)
|
|
|
|
for _, assertionName := range assertionNames {
|
|
assertion := component.Assertions[assertionName]
|
|
xmlSuites.TotalCount++
|
|
xmlSuite.TotalCount++
|
|
|
|
xmlCase := &TestCase{
|
|
ComponentName: componentName,
|
|
AssertionName: assertionName,
|
|
}
|
|
xmlSuite.Cases = append(xmlSuite.Cases, xmlCase)
|
|
|
|
switch assertion.Outcome {
|
|
case moduletest.Pending:
|
|
// We represent "pending" cases -- cases blocked by
|
|
// upstream errors -- as if they were "skipped" in JUnit
|
|
// terms, because we didn't actually check them and so
|
|
// can't say whether they succeeded or not.
|
|
xmlSuite.SkippedCount++
|
|
xmlCase.Skipped = &Outcome{
|
|
Message: assertion.Message,
|
|
}
|
|
case moduletest.Failed:
|
|
xmlSuites.FailureCount++
|
|
xmlSuite.FailureCount++
|
|
xmlCase.Failure = &Outcome{
|
|
Message: assertion.Message,
|
|
}
|
|
case moduletest.Error:
|
|
xmlSuites.ErrorCount++
|
|
xmlSuite.ErrorCount++
|
|
xmlCase.Error = &Outcome{
|
|
Message: assertion.Message,
|
|
}
|
|
|
|
// We'll also include the diagnostics in the "stderr"
|
|
// portion of the output, so they'll hopefully be visible
|
|
// in a test log viewer in JUnit-XML-Consuming CI systems.
|
|
var buf strings.Builder
|
|
for _, diag := range assertion.Diagnostics {
|
|
diagStr := format.DiagnosticPlain(diag, nil, 68)
|
|
buf.WriteString(diagStr)
|
|
}
|
|
xmlCase.Stderr = buf.String()
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
xmlOut, err := xml.MarshalIndent(&xmlSuites, "", " ")
|
|
if err != nil {
|
|
// If marshalling fails then that's a bug in the code above,
|
|
// because we should always be producing a value that is
|
|
// accepted by encoding/xml.
|
|
panic(fmt.Sprintf("invalid values to marshal as JUnit XML: %s", err))
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, xmlOut, 0644)
|
|
if err != nil {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Failed to write JUnit XML file",
|
|
fmt.Sprintf(
|
|
"Could not create %s to record the test results in JUnit XML format: %s.",
|
|
filename,
|
|
err,
|
|
),
|
|
))
|
|
}
|
|
|
|
return diags
|
|
}
|
|
|
|
func (v *testHuman) eprintRuleHeading(color, prefix, extra string) {
|
|
const lineCell string = "─"
|
|
textLen := len(prefix) + len(": ") + len(extra)
|
|
spacingLen := 2
|
|
leftLineLen := 3
|
|
|
|
rightLineLen := 0
|
|
width := v.streams.Stderr.Columns()
|
|
if (textLen + spacingLen + leftLineLen) < (width - 1) {
|
|
// (we allow an extra column at the end because some terminals can't
|
|
// print in the final column without wrapping to the next line)
|
|
rightLineLen = width - (textLen + spacingLen + leftLineLen) - 1
|
|
}
|
|
|
|
colorCode := "[" + color + "]"
|
|
|
|
// We'll prepare what we're going to print in memory first, so that we can
|
|
// send it all to stderr in one write in case other programs are also
|
|
// concurrently trying to write to the terminal for some reason.
|
|
var buf strings.Builder
|
|
buf.WriteString(v.colorize.Color(colorCode + strings.Repeat(lineCell, leftLineLen)))
|
|
buf.WriteByte(' ')
|
|
buf.WriteString(v.colorize.Color("[bold]" + colorCode + prefix + ":"))
|
|
buf.WriteByte(' ')
|
|
buf.WriteString(extra)
|
|
if rightLineLen > 0 {
|
|
buf.WriteByte(' ')
|
|
buf.WriteString(v.colorize.Color(colorCode + strings.Repeat(lineCell, rightLineLen)))
|
|
}
|
|
v.streams.Eprintln(buf.String())
|
|
}
|