Fix dependency handling (#179)

This change fixes dependency handling. The fix is coarse-grained but important for correctly ordering deletes. The implementation is currently lacking capabilities to track dependencies precisely through the TF black box.

What this does: 

- if dependencies are passed via dependsOn ResourceOption, Module ComponentResource now correctly depends on these

- if dependencies are passed implicitly with inputs, ModuleState Custom Resource now depends on all these dependencies. This means, in particular, that `tofu destroy` will now have to complete before the dependencies start getting destroyed themselves. This fixes issues such as #234 

- if Pulumi resources depend on any of the module's outputs, they now also will depend on ModuleState custom resource directly, and transitively on all the dependencies of the module's inputs. They should order their destroy operations accordingly. They also depend on the Component resource itself, respecting dependsOn option on that.

Fixes #151
Fixes #234

Author: t0yv0

pkg/modprovider/child.go

@@ -30,7 +30,7 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi/internals"
 	pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
 
-	"github.com/pulumi/pulumi-terraform-module/pkg/property"
+	"github.com/pulumi/pulumi-terraform-module/pkg/pulumix"
 	"github.com/pulumi/pulumi-terraform-module/pkg/tfsandbox"
 )
 
@@ -69,7 +69,7 @@ func newChildResource(
 	inputs := childResourceInputs(modUrn, sop.Address(), sop.Values())
 	t := childResourceTypeToken(pkgName, sop.Type())
 	name := childResourceName(sop)
-	inputsMap := property.MustUnmarshalPropertyMap(ctx, inputs)
+	inputsMap := pulumix.MustUnmarshalPropertyMap(ctx, inputs)
 	err := ctx.RegisterPackageResource(string(t), name, inputsMap, &resource, packageRef, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("RegisterResource failed for a child resource: %w", err)

pkg/modprovider/module_component.go

@@ -26,7 +26,7 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi/internals"
 
-	"github.com/pulumi/pulumi-terraform-module/pkg/property"
+	"github.com/pulumi/pulumi-terraform-module/pkg/pulumix"
 	"github.com/pulumi/pulumi-terraform-module/pkg/tfsandbox"
 )
 
@@ -52,7 +52,7 @@ func componentTypeToken(packageName packageName, compTypeName componentTypeName)
 	return tokens.Type(fmt.Sprintf("%s:index:%s", packageName, compTypeName))
 }
 
-func NewModuleComponentResource(
+func newModuleComponentResource(
 	ctx *pulumi.Context,
 	stateStore moduleStateStore,
 	planStore *planStore,
@@ -67,12 +67,12 @@ func NewModuleComponentResource(
 	providerSelfURN pulumi.URN,
 	providersConfig map[string]resource.PropertyMap,
 	opts ...pulumi.ResourceOption,
-) (componentUrn *urn.URN, outputs pulumi.Input, finalError error) {
+) (componentUrn *urn.URN, moduleStateResource *moduleStateResource, outputs pulumi.Map, finalError error) {
 	component := ModuleComponentResource{}
 	tok := componentTypeToken(pkgName, compTypeName)
 	err := ctx.RegisterComponentResource(string(tok), name, &component, opts...)
 	if err != nil {
-		return nil, nil, fmt.Errorf("RegisterComponentResource failed: %w", err)
+		return nil, nil, nil, fmt.Errorf("RegisterComponentResource failed: %w", err)
 	}
 
 	urn := component.MustURN(ctx.Context())
@@ -91,27 +91,25 @@ func NewModuleComponentResource(
 		providerSelfRef = newProviderSelfReference(ctx, providerSelfURN)
 	}
 
-	go func() {
-		resourceOptions := []pulumi.ResourceOption{
-			pulumi.Parent(&component),
-		}
+	resourceOptions := []pulumi.ResourceOption{
+		pulumi.Parent(&component),
+	}
 
-		if providerSelfRef != nil {
-			resourceOptions = append(resourceOptions, pulumi.Provider(providerSelfRef))
-		}
+	if providerSelfRef != nil {
+		resourceOptions = append(resourceOptions, pulumi.Provider(providerSelfRef))
+	}
 
-		_, err := newModuleStateResource(ctx,
-			// Needs to be prefixed by parent to avoid "duplicate URN".
-			fmt.Sprintf("%s-state", name),
-			pkgName,
-			urn,
-			packageRef,
-			moduleInputs,
-			resourceOptions...,
-		)
-
-		contract.AssertNoErrorf(err, "newModuleStateResource failed")
-	}()
+	modStateResource, err := newModuleStateResource(ctx,
+		// Needs to be prefixed by parent to avoid "duplicate URN".
+		fmt.Sprintf("%s-state", name),
+		pkgName,
+		urn,
+		packageRef,
+		moduleInputs,
+		resourceOptions...,
+	)
+
+	contract.AssertNoErrorf(err, "newModuleStateResource failed")
 
 	state := stateStore.AwaitOldState(urn)
 	defer func() {
@@ -126,7 +124,7 @@ func NewModuleComponentResource(
 	wd := tfsandbox.ModuleInstanceWorkdir(urn)
 	tf, err := tfsandbox.NewTofu(ctx.Context(), wd)
 	if err != nil {
-		return nil, nil, fmt.Errorf("Sandbox construction failed: %w", err)
+		return nil, nil, nil, fmt.Errorf("Sandbox construction failed: %w", err)
 	}
 
 	// Important: the name of the module instance in TF must be at least unique enough to
@@ -147,20 +145,20 @@ func NewModuleComponentResource(
 		moduleInputs, outputSpecs, providersConfig)
 
 	if err != nil {
-		return nil, nil, fmt.Errorf("Seed file generation failed: %w", err)
+		return nil, nil, nil, fmt.Errorf("Seed file generation failed: %w", err)
 	}
 
 	var moduleOutputs resource.PropertyMap
 	err = tf.PushStateAndLockFile(ctx.Context(), state.rawState, state.rawLockFile)
 	if err != nil {
-		return nil, nil, fmt.Errorf("PushStateAndLockFile failed: %w", err)
+		return nil, nil, nil, fmt.Errorf("PushStateAndLockFile failed: %w", err)
 	}
 
 	logger := newComponentLogger(ctx.Log, &component)
 
 	err = tf.Init(ctx.Context(), logger)
 	if err != nil {
-		return nil, nil, fmt.Errorf("Init failed: %w", err)
+		return nil, nil, nil, fmt.Errorf("Init failed: %w", err)
 	}
 
 	var childResources []*childResource
@@ -169,7 +167,7 @@ func NewModuleComponentResource(
 	// may be able to reuse the plan from DryRun for the subsequent application.
 	plan, err := tf.Plan(ctx.Context(), logger)
 	if err != nil {
-		return nil, nil, fmt.Errorf("Plan failed: %w", err)
+		return nil, nil, nil, fmt.Errorf("Plan failed: %w", err)
 	}
 
 	planStore.SetPlan(urn, plan)
@@ -203,21 +201,21 @@ func NewModuleComponentResource(
 			}
 		})
 		if err := errors.Join(errs...); err != nil {
-			return nil, nil, fmt.Errorf("Child resource init failed: %w", err)
+			return nil, nil, nil, fmt.Errorf("Child resource init failed: %w", err)
 		}
 		moduleOutputs = plan.Outputs()
 	} else {
 		// DryRun() = false corresponds to running pulumi up
 		tfState, err := tf.Apply(ctx.Context(), logger)
 		if err != nil {
-			return nil, nil, fmt.Errorf("Apply failed: %w", err)
+			return nil, nil, nil, fmt.Errorf("Apply failed: %w", err)
 		}
 
 		planStore.SetState(urn, tfState)
 
 		rawState, rawLockFile, err := tf.PullStateAndLockFile(ctx.Context())
 		if err != nil {
-			return nil, nil, fmt.Errorf("PullStateAndLockFile failed: %w", err)
+			return nil, nil, nil, fmt.Errorf("PullStateAndLockFile failed: %w", err)
 		}
 		state.rawState = rawState
 		state.rawLockFile = rawLockFile
@@ -246,7 +244,7 @@ func NewModuleComponentResource(
 			}
 		})
 		if err := errors.Join(errs...); err != nil {
-			return nil, nil, fmt.Errorf("Child resource init failed: %w", err)
+			return nil, nil, nil, fmt.Errorf("Child resource init failed: %w", err)
 		}
 
 		moduleOutputs = tfState.Outputs()
@@ -259,12 +257,12 @@ func NewModuleComponentResource(
 		cr.Await(ctx.Context())
 	}
 
-	marshalledOutputs := property.MustUnmarshalPropertyMap(ctx, moduleOutputs)
+	marshalledOutputs := pulumix.MustUnmarshalPropertyMap(ctx, moduleOutputs)
 	if err := ctx.RegisterResourceOutputs(&component, marshalledOutputs); err != nil {
-		return nil, nil, fmt.Errorf("RegisterResourceOutputs failed: %w", err)
+		return nil, nil, nil, fmt.Errorf("RegisterResourceOutputs failed: %w", err)
 	}
 
-	return &urn, marshalledOutputs, nil
+	return &urn, modStateResource, marshalledOutputs, nil
 }
 
 func newProviderSelfReference(ctx *pulumi.Context, urn1 pulumi.URN) pulumi.ProviderResource {

pkg/modprovider/server.go

@@ -35,6 +35,8 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 	pulumiprovider "github.com/pulumi/pulumi/sdk/v3/go/pulumi/provider"
 	pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
+
+	"github.com/pulumi/pulumi-terraform-module/pkg/pulumix"
 )
 
 func StartServer(hostClient *provider.HostClient) (pulumirpc.ResourceProviderServer, error) {
@@ -62,8 +64,12 @@ type server struct {
 	packageVersion       packageVersion
 	componentTypeName    componentTypeName
 	inferredModuleSchema *InferredModuleSchema
-	providerConfig       resource.PropertyMap
 	providerSelfURN      pulumi.URN
+
+	// Note that providerConfig does not include any first-class dependencies passed as Output values. In fact
+	// there are no Output values inside this map. In the current implementation this is OK as the data is only
+	// used to produce Terraform files to feed to opentofu and lacks the capability to track these dependencies.
+	providerConfig resource.PropertyMap
 }
 
 func (s *server) Parameterize(
@@ -212,9 +218,12 @@ func (s *server) Configure(
 	req *pulumirpc.ConfigureRequest,
 ) (*pulumirpc.ConfigureResponse, error) {
 	config, err := plugin.UnmarshalProperties(req.Args, plugin.MarshalOptions{
-		KeepUnknowns:     true,
-		RejectAssets:     true,
-		KeepSecrets:      true,
+		KeepUnknowns: true,
+		RejectAssets: true,
+		KeepSecrets:  true,
+
+		// This is only used to store s.providerConfig so it is OK to ignore dependencies in any Output values
+		// present in the request.
 		KeepOutputValues: false,
 	})
 
@@ -341,11 +350,10 @@ func (s *server) Construct(
 	req *pulumirpc.ConstructRequest,
 ) (*pulumirpc.ConstructResponse, error) {
 	inputProps, err := plugin.UnmarshalProperties(req.GetInputs(), plugin.MarshalOptions{
-		KeepUnknowns:  true,
-		KeepSecrets:   true,
-		KeepResources: true,
-		// TODO[https://github.com/pulumi/pulumi-terraform-module/issues/151] support Outputs in Unmarshal
-		KeepOutputValues: false,
+		KeepUnknowns:     true,
+		KeepSecrets:      true,
+		KeepResources:    true,
+		KeepOutputValues: true,
 	})
 
 	providersConfig := cleanProvidersConfig(s.providerConfig)
@@ -361,12 +369,12 @@ func (s *server) Construct(
 	return pulumiprovider.Construct(ctx, req, s.hostClient.EngineConn(), func(
 		ctx *pulumi.Context, typ, name string,
 		_ pulumiprovider.ConstructInputs,
-		_ pulumi.ResourceOption,
+		resourceOptions pulumi.ResourceOption,
 	) (*pulumiprovider.ConstructResult, error) {
 		ctok := componentTypeToken(s.packageName, s.componentTypeName)
 		switch typ {
 		case string(ctok):
-			componentUrn, outputs, err := NewModuleComponentResource(ctx,
+			componentUrn, modStateResource, outputs, err := newModuleComponentResource(ctx,
 				s.stateStore,
 				s.planStore,
 				s.packageName,
@@ -378,14 +386,20 @@ func (s *server) Construct(
 				s.inferredModuleSchema,
 				packageRef,
 				s.providerSelfURN,
-				providersConfig)
+				providersConfig,
+				resourceOptions,
+			)
 
 			if err != nil {
 				return nil, fmt.Errorf("NewModuleComponentResource failed: %w", err)
 			}
+
 			constructResult := &pulumiprovider.ConstructResult{
-				URN:   pulumi.URN(string(*componentUrn)),
-				State: outputs,
+				URN: pulumi.URN(string(*componentUrn)),
+				// Every Output needs to depend on the modStateResource.
+				State: pulumix.MapWithBroadcastDependencies(ctx.Context(), []pulumi.Resource{
+					modStateResource,
+				}, outputs),
 			}
 			return constructResult, nil
 		default:
@@ -415,9 +429,12 @@ func (s *server) CheckConfig(
 	s.providerSelfURN = pulumi.URN(req.Urn)
 
 	config, err := plugin.UnmarshalProperties(req.GetNews(), plugin.MarshalOptions{
-		KeepUnknowns:     true,
-		RejectAssets:     true,
-		KeepSecrets:      true,
+		KeepUnknowns: true,
+		RejectAssets: true,
+		KeepSecrets:  true,
+
+		// This is only used to store s.providerConfig so it is OK to ignore dependencies in any Output values
+		// present in the request.
 		KeepOutputValues: false,
 	})
 

pkg/modprovider/state.go

@@ -31,7 +31,7 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 	pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
 
-	"github.com/pulumi/pulumi-terraform-module/pkg/property"
+	"github.com/pulumi/pulumi-terraform-module/pkg/pulumix"
 	"github.com/pulumi/pulumi-terraform-module/pkg/tfsandbox"
 )
 
@@ -119,7 +119,7 @@ func newModuleStateResource(
 	var res moduleStateResource
 	tok := moduleStateTypeToken(pkgName)
 
-	inputsMap := property.MustUnmarshalPropertyMap(ctx, resource.PropertyMap{
+	inputsMap := pulumix.MustUnmarshalPropertyMap(ctx, resource.PropertyMap{
 		moduleURNPropName: resource.NewStringProperty(string(modUrn)),
 		"moduleInputs":    resource.NewObjectProperty(moduleInputs),
 	})
@@ -268,7 +268,11 @@ func (h *moduleStateHandler) Delete(
 		KeepUnknowns:  true,
 		KeepSecrets:   true,
 		KeepResources: true,
-		// TODO[pulumi/pulumi-terraform-module#151] support Outputs in Unmarshal
+
+		// If there are any resource.NewOutputProperty values in old inputs with dependencies, this setting
+		// will ignore the dependencies and remove these values in favor of simpler Computed or Secret values.
+		// This is OK for the purposes of Delete running tofu destroy because the code cannot take advantage of
+		// these precisely tracked dependencies here anyway. So it is OK to ignore them.
 		KeepOutputValues: false,
 	})
 	if err != nil {

pkg/pulumix/map.go

@@ -0,0 +1,34 @@
+// Copyright 2016-2025, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pulumix
+
+import (
+	"context"
+
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+)
+
+// Constructs an updated Map where every Input is augmented with the same set of additional dependencies.
+func MapWithBroadcastDependencies(ctx context.Context, dependencies []pulumi.Resource, out pulumi.Map) pulumi.Map {
+	if len(dependencies) == 0 {
+		return out
+	}
+	result := pulumi.Map{}
+	for k, input := range out {
+		output := pulumi.ToOutputWithContext(ctx, input)
+		result[k] = pulumi.OutputWithDependencies(ctx, output, dependencies...)
+	}
+	return result
+}

pkg/property/package.go pkg/pulumix/package.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package property exposes PropertyMap/PropertyValue helpers that did not make it into the official Pulumi Go SDK yet.
+// Package pulumix exposes helpers that did not make it into the official Pulumi Go SDK yet.
 //
 // See also: https://github.com/pulumi/pulumi/issues/18447
-package property
+package pulumix