Implement QueryJSON and introduce new way for consuming Terraform's structured logging (#539)

Co-authored-by: Radek Simko <[email protected]>
Co-authored-by: Sarah French <[email protected]>

Author: 3 people

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/google/go-cmp v0.7.0
 	github.com/hashicorp/go-version v1.7.0
 	github.com/hashicorp/hc-install v0.9.2
-	github.com/hashicorp/terraform-json v0.26.0
+	github.com/hashicorp/terraform-json v0.27.1
 	github.com/zclconf/go-cty v1.16.4
 	github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b
 )

go.sum

@@ -51,8 +51,8 @@ github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKe
 github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/hc-install v0.9.2 h1:v80EtNX4fCVHqzL9Lg/2xkp62bbvQMnvPQ0G+OmtO24=
 github.com/hashicorp/hc-install v0.9.2/go.mod h1:XUqBQNnuT4RsxoxiM9ZaUk0NX8hi2h+Lb6/c0OZnC/I=
-github.com/hashicorp/terraform-json v0.26.0 h1:+BnJavhRH+oyNWPnfzrfQwVWCZBFMvjdiH2Vi38Udz4=
-github.com/hashicorp/terraform-json v0.26.0/go.mod h1:eyWCeC3nrZamyrKLFnrvwpc3LQPIJsx8hWHQ/nu2/v4=
+github.com/hashicorp/terraform-json v0.27.1 h1:zWhEracxJW6lcjt/JvximOYyc12pS/gaKSy/wzzE7nY=
+github.com/hashicorp/terraform-json v0.27.1/go.mod h1:GzPLJ1PLdUG5xL6xn1OXWIjteQRT2CNT9o/6A9mi9hE=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=

tfexec/cmd.go

@@ -12,12 +12,14 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"iter"
 	"os"
 	"os/exec"
 	"runtime"
 	"strings"
 
 	"github.com/hashicorp/terraform-exec/internal/version"
+	tfjson "github.com/hashicorp/terraform-json"
 )
 
 const (
@@ -216,6 +218,72 @@ func (tf *Terraform) runTerraformCmdJSON(ctx context.Context, cmd *exec.Cmd, v i
 	return dec.Decode(v)
 }
 
+func (tf *Terraform) runTerraformCmdJSONLog(ctx context.Context, cmd *exec.Cmd) iter.Seq[NextMessage] {
+	pr, pw := io.Pipe()
+	tf.SetStdout(pw)
+
+	emitter := newLogMsgEmitter(pr)
+
+	go func() {
+		err := tf.runTerraformCmd(ctx, cmd)
+		emitter.done <- errors.Join(err, pw.Close())
+	}()
+
+	return func(yield func(msg NextMessage) bool) {
+		for {
+			nextMsg := emitter.NextMessage()
+			ok := yield(nextMsg)
+			if !ok || nextMsg.Msg == nil {
+				return
+			}
+		}
+	}
+}
+
+func newLogMsgEmitter(stdoutReader io.ReadCloser) *logMsgEmitter {
+	return &logMsgEmitter{
+		scanner:      bufio.NewScanner(stdoutReader),
+		stdoutReader: stdoutReader,
+		done:         make(chan error, 1),
+	}
+}
+
+type logMsgEmitter struct {
+	scanner      *bufio.Scanner
+	stdoutReader io.Closer
+	done         chan error
+}
+
+type NextMessage struct {
+	Msg tfjson.LogMsg
+	Err error
+}
+
+// NextMessage returns next decoded message, if any, along with any errors.
+// Stdout reader is closed when the last message is received.
+//
+// Error returned can be related to decoding of the message, the Terraform command
+// or closing of stdout reader.
+//
+// Any error coming from Terraform (such as wrong configuration syntax) is
+// represented as LogMsg of Level [tfjson.Error].
+func (e *logMsgEmitter) NextMessage() NextMessage {
+	if e.scanner.Scan() {
+		msg, err := tfjson.UnmarshalLogMessage(e.scanner.Bytes())
+		return NextMessage{
+			Msg: msg,
+			Err: err,
+		}
+	}
+
+	err := <-e.done
+	err = errors.Join(err, e.scanner.Err(), e.stdoutReader.Close())
+	return NextMessage{
+		Msg: nil,
+		Err: err,
+	}
+}
+
 // mergeUserAgent does some minor deduplication to ensure we aren't
 // just using the same append string over and over.
 func mergeUserAgent(uas ...string) string {

tfexec/internal/e2etest/query_test.go

@@ -0,0 +1,82 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package e2etest
+
+import (
+	"context"
+	"regexp"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/hashicorp/go-version"
+	"github.com/hashicorp/terraform-exec/tfexec"
+	"github.com/hashicorp/terraform-exec/tfexec/internal/testutil"
+	tfjson "github.com/hashicorp/terraform-json"
+)
+
+func TestQueryJSON_TF112(t *testing.T) {
+	versions := []string{testutil.Latest_v1_12}
+
+	runTestWithVersions(t, versions, "query", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
+		err := tf.Init(context.Background())
+		if err != nil {
+			t.Fatalf("error running Init in test directory: %s", err)
+		}
+
+		re := regexp.MustCompile("terraform query -json was added in 1.14.0")
+
+		_, err = tf.QueryJSON(context.Background())
+		if err != nil && !re.MatchString(err.Error()) {
+			t.Fatalf("error running Query: %s", err)
+		}
+	})
+}
+
+func TestQueryJSON_TF114(t *testing.T) {
+	versions := []string{testutil.Latest_Alpha_v1_14}
+
+	runTestWithVersions(t, versions, "query", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
+		err := tf.Init(context.Background())
+		if err != nil {
+			t.Fatalf("error running Init in test directory: %s", err)
+		}
+
+		iter, err := tf.QueryJSON(context.Background())
+		if err != nil {
+			t.Fatalf("error running Query: %s", err)
+		}
+
+		results := 0
+		listingStarted := 0
+		var completeData tfjson.ListCompleteData
+		for nextMsg := range iter {
+			if nextMsg.Err != nil {
+				t.Fatalf("error getting next message: %s", err)
+			}
+			switch m := nextMsg.Msg.(type) {
+			case tfjson.ListStartMessage:
+				listingStarted++
+			case tfjson.ListResourceFoundMessage:
+				results++
+			case tfjson.ListCompleteMessage:
+				completeData = m.ListComplete
+			}
+		}
+
+		if listingStarted != 1 {
+			t.Fatalf("expected exactly 1 list start message, got %d", listingStarted)
+		}
+		if results != 5 {
+			t.Fatalf("expected 5 query results, got %d", results)
+		}
+		expectedData := tfjson.ListCompleteData{
+			Address:      "list.concept_pet.pets",
+			ResourceType: "concept_pet",
+			Total:        5,
+		}
+		if diff := cmp.Diff(expectedData, completeData); diff != "" {
+			t.Fatalf("unexpected complete message data: %s", diff)
+		}
+	})
+}

tfexec/internal/e2etest/testdata/query/main.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    concept = {
+      source  = "dbanck/concept"
+      version = "0.1.0"
+    }
+  }
+}

tfexec/internal/e2etest/testdata/query/main.tfquery.hcl

@@ -0,0 +1,3 @@
+list "concept_pet" "pets" {
+    provider = concept
+}

tfexec/internal/testutil/tfcache.go

@@ -36,6 +36,7 @@ const (
 	Latest_Alpha_v1_10 = "1.10.0-alpha20240926"
 	Latest_v1_11       = "1.11.4"
 	Latest_v1_12       = "1.12.2"
+	Latest_Alpha_v1_14 = "1.14.0-alpha20250903"
 )
 
 const appendUserAgent = "tfexec-testutil"

tfexec/options.go

@@ -184,6 +184,14 @@ func FromModule(source string) *FromModuleOption {
 	return &FromModuleOption{source}
 }
 
+type GenerateConfigOutOption struct {
+	path string
+}
+
+func GenerateConfigOut(path string) *GenerateConfigOutOption {
+	return &GenerateConfigOutOption{path}
+}
+
 type GetOption struct {
 	get bool
 }

tfexec/query.go

@@ -0,0 +1,127 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package tfexec
+
+import (
+	"context"
+	"fmt"
+	"iter"
+	"os/exec"
+)
+
+type queryConfig struct {
+	dir            string
+	generateConfig string
+	reattachInfo   ReattachInfo
+	vars           []string
+	varFiles       []string
+}
+
+var defaultQueryOptions = queryConfig{}
+
+// QueryOption represents options used in the Query method.
+type QueryOption interface {
+	configureQuery(*queryConfig)
+}
+
+func (opt *DirOption) configureQuery(conf *queryConfig) {
+	conf.dir = opt.path
+}
+
+func (opt *GenerateConfigOutOption) configureQuery(conf *queryConfig) {
+	conf.generateConfig = opt.path
+}
+
+func (opt *ReattachOption) configureQuery(conf *queryConfig) {
+	conf.reattachInfo = opt.info
+}
+
+func (opt *VarFileOption) configureQuery(conf *queryConfig) {
+	conf.varFiles = append(conf.varFiles, opt.path)
+}
+
+func (opt *VarOption) configureQuery(conf *queryConfig) {
+	conf.vars = append(conf.vars, opt.assignment)
+}
+
+// QueryJSON executes `terraform query` with the specified options as well as the
+// `-json` flag and waits for it to complete.
+//
+// Using the `-json` flag will result in
+// [machine-readable](https://developer.hashicorp.com/terraform/internals/machine-readable-ui)
+// JSON being written to the supplied `io.Writer`.
+//
+// The returned error is nil if `terraform query` has been executed and exits
+// with 0.
+//
+// QueryJSON is likely to be removed in a future major version in favour of
+// query returning JSON by default.
+func (tf *Terraform) QueryJSON(ctx context.Context, opts ...QueryOption) (iter.Seq[NextMessage], error) {
+	err := tf.compatible(ctx, tf1_14_0, nil)
+	if err != nil {
+		return nil, fmt.Errorf("terraform query -json was added in 1.14.0: %w", err)
+	}
+
+	queryCmd, err := tf.queryJSONCmd(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return tf.runTerraformCmdJSONLog(ctx, queryCmd), nil
+}
+
+func (tf *Terraform) queryJSONCmd(ctx context.Context, opts ...QueryOption) (*exec.Cmd, error) {
+	c := defaultQueryOptions
+
+	for _, o := range opts {
+		o.configureQuery(&c)
+	}
+
+	args, err := tf.buildQueryArgs(ctx, c)
+	if err != nil {
+		return nil, err
+	}
+
+	args = append(args, "-json")
+
+	return tf.buildQueryCmd(ctx, c, args)
+}
+
+func (tf *Terraform) buildQueryArgs(ctx context.Context, c queryConfig) ([]string, error) {
+	args := []string{"query", "-no-color"}
+
+	if c.generateConfig != "" {
+		args = append(args, "-generate-config-out="+c.generateConfig)
+	}
+
+	for _, vf := range c.varFiles {
+		args = append(args, "-var-file="+vf)
+	}
+
+	if c.vars != nil {
+		for _, v := range c.vars {
+			args = append(args, "-var", v)
+		}
+	}
+
+	return args, nil
+}
+
+func (tf *Terraform) buildQueryCmd(ctx context.Context, c queryConfig, args []string) (*exec.Cmd, error) {
+	// optional positional argument
+	if c.dir != "" {
+		args = append(args, c.dir)
+	}
+
+	mergeEnv := map[string]string{}
+	if c.reattachInfo != nil {
+		reattachStr, err := c.reattachInfo.marshalString()
+		if err != nil {
+			return nil, err
+		}
+		mergeEnv[reattachEnvVar] = reattachStr
+	}
+
+	return tf.buildTerraformCmd(ctx, mergeEnv, args...), nil
+}