Merge pull request #20242 from hashicorp/svh/b-scanner-buffer-v0.12

backend/remote: fix bufio.Scanner: token too long

backend/remote/backend_apply.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"log"
 
 	tfe "github.com/hashicorp/go-tfe"
@@ -202,21 +203,34 @@ func (b *Remote) opApply(stopCtx, cancelCtx context.Context, op *backend.Operati
 	if err != nil {
 		return r, generalError("Failed to retrieve logs", err)
 	}
-	scanner := bufio.NewScanner(logs)
+	reader := bufio.NewReaderSize(logs, 64*1024)
 
-	skip := 0
+	if b.CLI != nil {
-	for scanner.Scan() {
+		skip := 0
-		// Skip the first 3 lines to prevent duplicate output.
+		for next := true; next; {
-		if skip < 3 {
+			var l, line []byte
-			skip++
+
-			continue
+			for isPrefix := true; isPrefix; {
+				l, isPrefix, err = reader.ReadLine()
+				if err != nil {
+					if err != io.EOF {
+						return r, generalError("Failed to read logs", err)
+					}
+					next = false
+				}
+				line = append(line, l...)
+			}
+
+			// Skip the first 3 lines to prevent duplicate output.
+			if skip < 3 {
+				skip++
+				continue
+			}
+
+			if next || len(line) > 0 {
+				b.CLI.Output(b.Colorize().Color(string(line)))
+			}
 		}
-		if b.CLI != nil {
-			b.CLI.Output(b.Colorize().Color(scanner.Text()))
-		}
-	}
-	if err := scanner.Err(); err != nil {
-		return r, generalError("Failed to read logs", err)
 	}
 
 	return r, nil

backend/remote/backend_common.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"math"
 	"time"
 
@@ -227,7 +228,7 @@ func (b *Remote) checkPolicy(stopCtx, cancelCtx context.Context, op *backend.Ope
 		if err != nil {
 			return generalError("Failed to retrieve policy check logs", err)
 		}
-		scanner := bufio.NewScanner(logs)
+		reader := bufio.NewReaderSize(logs, 64*1024)
 
 		// Retrieve the policy check to get its current status.
 		pc, err := b.client.PolicyChecks.Read(stopCtx, pc.ID)
@@ -249,14 +250,26 @@ func (b *Remote) checkPolicy(stopCtx, cancelCtx context.Context, op *backend.Ope
 			b.CLI.Output(b.Colorize().Color(msgPrefix + ":\n"))
 		}
 
-		for scanner.Scan() {
+		if b.CLI != nil {
-			if b.CLI != nil {
+			for next := true; next; {
-				b.CLI.Output(b.Colorize().Color(scanner.Text()))
+				var l, line []byte
+
+				for isPrefix := true; isPrefix; {
+					l, isPrefix, err = reader.ReadLine()
+					if err != nil {
+						if err != io.EOF {
+							return generalError("Failed to read logs", err)
+						}
+						next = false
+					}
+					line = append(line, l...)
+				}
+
+				if next || len(line) > 0 {
+					b.CLI.Output(b.Colorize().Color(string(line)))
+				}
 			}
 		}
-		if err := scanner.Err(); err != nil {
-			return generalError("Failed to read logs", err)
-		}
 
 		switch pc.Status {
 		case tfe.PolicyPasses:

backend/remote/backend_plan.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
@@ -253,16 +254,28 @@ func (b *Remote) plan(stopCtx, cancelCtx context.Context, op *backend.Operation,
 	if err != nil {
 		return r, generalError("Failed to retrieve logs", err)
 	}
-	scanner := bufio.NewScanner(logs)
+	reader := bufio.NewReaderSize(logs, 64*1024)
 
-	for scanner.Scan() {
+	if b.CLI != nil {
-		if b.CLI != nil {
+		for next := true; next; {
-			b.CLI.Output(b.Colorize().Color(scanner.Text()))
+			var l, line []byte
+
+			for isPrefix := true; isPrefix; {
+				l, isPrefix, err = reader.ReadLine()
+				if err != nil {
+					if err != io.EOF {
+						return r, generalError("Failed to read logs", err)
+					}
+					next = false
+				}
+				line = append(line, l...)
+			}
+
+			if next || len(line) > 0 {
+				b.CLI.Output(b.Colorize().Color(string(line)))
+			}
 		}
 	}
-	if err := scanner.Err(); err != nil {
-		return r, generalError("Failed to read logs", err)
-	}
 
 	// Retrieve the run to get its current status.
 	r, err = b.client.Runs.Read(stopCtx, r.ID)

backend/remote/backend_plan_test.go

@@ -63,6 +63,36 @@ func TestRemote_planBasic(t *testing.T) {
 	}
 }
 
+func TestRemote_planLongLine(t *testing.T) {
+	b := testBackendDefault(t)
+
+	op, configCleanup := testOperationPlan(t, "./test-fixtures/plan-long-line")
+	defer configCleanup()
+
+	op.Workspace = backend.DefaultStateName
+
+	run, err := b.Operation(context.Background(), op)
+	if err != nil {
+		t.Fatalf("error starting operation: %v", err)
+	}
+
+	<-run.Done()
+	if run.Result != backend.OperationSuccess {
+		t.Fatalf("operation failed: %s", b.CLI.(*cli.MockUi).ErrorWriter.String())
+	}
+	if run.PlanEmpty {
+		t.Fatal("expected a non-empty plan")
+	}
+
+	output := b.CLI.(*cli.MockUi).OutputWriter.String()
+	if !strings.Contains(output, "Running plan in the remote backend") {
+		t.Fatalf("expected remote backend header in output: %s", output)
+	}
+	if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") {
+		t.Fatalf("expected plan summery in output: %s", output)
+	}
+}
+
 func TestRemote_planWithoutPermissions(t *testing.T) {
 	b, bCleanup := testBackendNoDefault(t)
 	defer bCleanup()