Save tofu lockfile to preserve provider choices (#184)

This PR adds the Tofu
[lockfile](https://opentofu.org/docs/language/files/dependency-lock/) to
the ModuleState resource. Now whenever we push/pull state we also
push/pull the lockfile. Using the lockfile will ensure that every time
we run tofu we will always select the same version of the providers.

closes #115

Author: corymhall

pkg/modprovider/module_component.go

@@ -151,14 +151,14 @@ func NewModuleComponentResource(
 	}
 
 	var moduleOutputs resource.PropertyMap
-	err = tf.Init(ctx.Context())
+	err = tf.PushStateAndLockFile(ctx.Context(), state.rawState, state.rawLockFile)
 	if err != nil {
-		return nil, nil, fmt.Errorf("Init failed: %w", err)
+		return nil, nil, fmt.Errorf("PushStateAndLockFile failed: %w", err)
 	}
 
-	err = tf.PushState(ctx.Context(), state.rawState)
+	err = tf.Init(ctx.Context())
 	if err != nil {
-		return nil, nil, fmt.Errorf("PushState failed: %w", err)
+		return nil, nil, fmt.Errorf("Init failed: %w", err)
 	}
 
 	var childResources []*childResource
@@ -219,14 +219,12 @@ func NewModuleComponentResource(
 
 		planStore.SetState(urn, tfState)
 
-		rawState, ok, err := tf.PullState(ctx.Context())
+		rawState, rawLockFile, err := tf.PullStateAndLockFile(ctx.Context())
 		if err != nil {
-			return nil, nil, fmt.Errorf("PullState failed: %w", err)
-		}
-		if !ok {
-			return nil, nil, errors.New("PullState did not find state")
+			return nil, nil, fmt.Errorf("PullStateAndLockFile failed: %w", err)
 		}
 		state.rawState = rawState
+		state.rawLockFile = rawLockFile
 
 		// Make sure child resources can read updated state.
 		stateStore.SetNewState(urn, state)

pkg/modprovider/state.go

@@ -17,7 +17,6 @@ package modprovider
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 
 	"google.golang.org/protobuf/types/known/emptypb"
@@ -45,10 +44,12 @@ const (
 type moduleState struct {
 	// Intended to store contents of TF state exactly.
 	rawState []byte
+	// Intended to store contents of TF lock file exactly.
+	rawLockFile []byte
 }
 
 func (ms *moduleState) Equal(other moduleState) bool {
-	return bytes.Equal(ms.rawState, other.rawState)
+	return bytes.Equal(ms.rawState, other.rawState) && bytes.Equal(ms.rawLockFile, other.rawLockFile)
 }
 
 func (ms *moduleState) Unmarshal(s *structpb.Struct) {
@@ -63,6 +64,10 @@ func (ms *moduleState) Unmarshal(s *structpb.Struct) {
 	if !ok {
 		return // empty
 	}
+	if lock, ok := props["lock"]; ok {
+		lockString := lock.StringValue()
+		ms.rawLockFile = []byte(lockString)
+	}
 	stateString := state.StringValue()
 	ms.rawState = []byte(stateString)
 }
@@ -71,6 +76,7 @@ func (ms *moduleState) Marshal() *structpb.Struct {
 	state := resource.PropertyMap{
 		// TODO[pulumi/pulumi-terraform-module#148] store as JSON-y map
 		"state": resource.MakeSecret(resource.NewStringProperty(string(ms.rawState))),
+		"lock":  resource.NewStringProperty(string(ms.rawLockFile)),
 	}
 
 	value, err := plugin.MarshalProperties(state, plugin.MarshalOptions{
@@ -281,14 +287,14 @@ func (h *moduleStateHandler) Delete(
 		return nil, fmt.Errorf("Seed file generation failed: %w", err)
 	}
 
-	err = tf.Init(ctx)
+	err = tf.PushStateAndLockFile(ctx, oldState.rawState, oldState.rawLockFile)
 	if err != nil {
-		return nil, fmt.Errorf("Init failed: %w", err)
+		return nil, fmt.Errorf("PushStateAndLockFile failed: %w", err)
 	}
 
-	err = tf.PushState(ctx, oldState.rawState)
+	err = tf.Init(ctx)
 	if err != nil {
-		return nil, fmt.Errorf("PushState failed: %w", err)
+		return nil, fmt.Errorf("Init failed: %w", err)
 	}
 
 	err = tf.Destroy(ctx)
@@ -343,9 +349,9 @@ func (h *moduleStateHandler) Read(
 	oldState := moduleState{}
 	oldState.Unmarshal(req.GetProperties())
 
-	err = tf.PushState(ctx, oldState.rawState)
+	err = tf.PushStateAndLockFile(ctx, oldState.rawState, oldState.rawLockFile)
 	if err != nil {
-		return nil, fmt.Errorf("PushState failed: %w", err)
+		return nil, fmt.Errorf("PushStateAndLockFile failed: %w", err)
 	}
 
 	plan, err := tf.PlanRefreshOnly(ctx)
@@ -365,16 +371,14 @@ func (h *moduleStateHandler) Read(
 	// Child resources need to access the state in their Read() implementation.
 	h.planStore.SetState(modUrn, state)
 
-	rawState, ok, err := tf.PullState(ctx)
+	rawState, rawLockFile, err := tf.PullStateAndLockFile(ctx)
 	if err != nil {
-		return nil, fmt.Errorf("PullState failed: %w", err)
-	}
-	if !ok {
-		return nil, errors.New("PullState did not find state")
+		return nil, fmt.Errorf("PullStateAndLockFile failed: %w", err)
 	}
 
 	refreshedModuleState := moduleState{
-		rawState: rawState,
+		rawState:    rawState,
+		rawLockFile: rawLockFile,
 	}
 
 	// The engine will call Diff() after Read(), and it would expect this to be populated.

pkg/tfsandbox/state.go

@@ -23,23 +23,50 @@ import (
 )
 
 const defaultStateFile = "terraform.tfstate"
+const defaultLockFile = ".terraform.lock.hcl"
 
-func (t *Tofu) PullState(_ context.Context) (json.RawMessage, bool, error) {
+// PullStateAndLockFile reads the state and lock file from the Tofu working directory.
+// If the lock file is not present, it returns nil for the lock file and no error.
+// It's possible for modules to not have any providers which would mean no lock file
+func (t *Tofu) PullStateAndLockFile(_ context.Context) (state json.RawMessage, lockFile []byte, err error) {
+	state, err = t.pullState(context.Background())
+	if err != nil {
+		return nil, nil, err
+	}
+	lockFile, err = t.pullLockFile(context.Background())
+	if err != nil {
+		return nil, nil, err
+	}
+	return state, lockFile, nil
+}
+
+// PushStateAndLockFile writes the state and lock file to the Tofu working directory.
+func (t *Tofu) PushStateAndLockFile(_ context.Context, state json.RawMessage, lock []byte) error {
+	if err := t.pushState(context.Background(), state); err != nil {
+		return err
+	}
+	if err := t.pushLockFile(context.Background(), lock); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *Tofu) pullState(_ context.Context) (json.RawMessage, error) {
 	// If for some reason this needs to work in contexts with a non-default state provider, or
 	// take advantage of built-in locking, then tofu state pull command can be used instead.
 	path := filepath.Join(t.WorkingDir(), defaultStateFile)
 	bytes, err := os.ReadFile(path)
 	switch {
 	case err != nil && os.IsNotExist(err):
-		return nil, false, nil
+		return nil, fmt.Errorf("default tfstate file not found: %w", err)
 	case err != nil:
-		return nil, false, fmt.Errorf("failed to read the default tfstate file: %w", err)
+		return nil, fmt.Errorf("failed to read the default tfstate file: %w", err)
 	default:
-		return json.RawMessage(bytes), true, nil
+		return json.RawMessage(bytes), nil
 	}
 }
 
-func (t *Tofu) PushState(_ context.Context, data json.RawMessage) error {
+func (t *Tofu) pushState(_ context.Context, data json.RawMessage) error {
 	// If for some reason this needs to work in contexts with a non-default state provider, or
 	// take advantage of built-in locking, then tofu state push command can be used instead.
 	path := filepath.Join(t.WorkingDir(), defaultStateFile)
@@ -48,3 +75,28 @@ func (t *Tofu) PushState(_ context.Context, data json.RawMessage) error {
 	}
 	return nil
 }
+
+func (t *Tofu) pullLockFile(_ context.Context) ([]byte, error) {
+	path := filepath.Join(t.WorkingDir(), defaultLockFile)
+	bytes, err := os.ReadFile(path)
+	switch {
+	// If the lock file is not present, that's fine
+	case err != nil && os.IsNotExist(err):
+		return nil, nil
+	case err != nil:
+		return nil, fmt.Errorf("failed to read the default lock file: %w", err)
+	default:
+		return bytes, nil
+	}
+}
+
+func (t *Tofu) pushLockFile(_ context.Context, data []byte) error {
+	if data == nil {
+		return nil
+	}
+	path := filepath.Join(t.WorkingDir(), defaultLockFile)
+	if err := os.WriteFile(path, data, 0600); err != nil {
+		return fmt.Errorf("failed to write the default lock file: %w", err)
+	}
+	return nil
+}

pkg/tfsandbox/state_test.go

@@ -73,9 +73,8 @@ func TestState(t *testing.T) {
 		resource.PropertyKey("statically_known"): resource.NewStringProperty("static value"),
 	}, moduleOutputs)
 
-	rawState, ok, err := tofu.PullState(ctx)
+	rawState, rawLockFile, err := tofu.PullStateAndLockFile(ctx)
 	require.NoError(t, err, "error pulling tofu state")
-	require.True(t, ok, "no tofu state found")
 
 	type stateModel struct {
 		Resources []any `json:"resources"`
@@ -97,7 +96,7 @@ func TestState(t *testing.T) {
 	// Now modify the state and run a plan.
 
 	newState := bytes.ReplaceAll(rawState, []byte(`"test"`), []byte(`"test2"`))
-	err = tofu.PushState(ctx, newState)
+	err = tofu.PushStateAndLockFile(ctx, newState, rawLockFile)
 	require.NoError(t, err, "error pushing tofu state")
 
 	plan, err := tofu.Plan(ctx)