Allow backend state files to not include version data when a builtin or reattached provider is in use.

Author: SarahFrench

internal/command/meta_backend.go

@@ -40,6 +40,7 @@ import (
 	"github.com/hashicorp/terraform/internal/depsfile"
 	"github.com/hashicorp/terraform/internal/didyoumean"
 	"github.com/hashicorp/terraform/internal/getproviders/providerreqs"
+	"github.com/hashicorp/terraform/internal/getproviders/reattach"
 	"github.com/hashicorp/terraform/internal/plans"
 	"github.com/hashicorp/terraform/internal/providers"
 	"github.com/hashicorp/terraform/internal/states"
@@ -1667,42 +1668,26 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, stateStoreHash int, provide
 		s = workdir.NewBackendStateFile()
 	}
 
-	var pVersion *version.Version
-	if c.ProviderAddr.Equals(addrs.NewBuiltInProvider("terraform")) {
-		// If we're handling the builtin "terraform" provider then there's no version information to store in the dependency lock file, so don't access it.
-		// We must record a value into the backend state file, and we cannot include a value that changes (e.g. the Terraform core binary version) as migration
-		// is impossible with builtin providers.
-		// So, we use an arbitrary stand-in version.
-		standInVersion, err := version.NewVersion("0.0.1")
-		if err != nil {
-			diags = diags.Append(fmt.Errorf("Error when creating a backend state file. This is a bug in Terraform and should be reported: %w",
-				err))
-			return nil, diags
-		}
-		pVersion = standInVersion
+	var pVersion *version.Version // This will remain nil for builtin providers or unmanaged providers.
+	if c.ProviderAddr.Hostname == addrs.BuiltInProviderHost {
+		diags = diags.Append(&hcl.Diagnostic{
+			Severity: hcl.DiagWarning,
+			Summary:  "State storage is using a builtin provider",
+			Detail:   "Terraform is using a builtin provider for initializing state storage. Terraform will be less able to detect when state migrations are required in future init commands.",
+		})
 	} else {
-		isReattached, err := isProviderReattached(c.ProviderAddr)
+		isReattached, err := reattach.IsProviderReattached(c.ProviderAddr, os.Getenv("TF_REATTACH_PROVIDERS"))
 		if err != nil {
 			diags = diags.Append(fmt.Errorf("Error determining if the state storage provider is reattached or not. This is a bug in Terraform and should be reported: %w",
 				err))
 			return nil, diags
 		}
 		if isReattached {
-			// If the provider is unmanaged then it won't be in the locks.
-			// If there are no locks then there's no version information to for us to access and use when creating the backend state file.
-			// So, we use an arbitrary stand-in version.
 			diags = diags.Append(&hcl.Diagnostic{
 				Severity: hcl.DiagWarning,
 				Summary:  "State storage provider is not managed by Terraform",
-				Detail:   "Terraform is using a provider supplied via TF_REATTACH_PROVIDERS for initializing state storage. This will affect Terraform's ability to detect when state migrations are required.",
+				Detail:   "Terraform is using a provider supplied via TF_REATTACH_PROVIDERS for initializing state storage. Terraform will be less able to detect when state migrations are required in future init commands.",
 			})
-			standInVersion, err := version.NewVersion("0.0.1")
-			if err != nil {
-				diags = diags.Append(fmt.Errorf("Error when creating a backend state file. This is a bug in Terraform and should be reported: %w",
-					err))
-				return nil, diags
-			}
-			pVersion = standInVersion
 		} else {
 			// The provider is not built in and is being managed by Terraform
 			// This is the most common scenario, by far.
@@ -1804,29 +1789,6 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, stateStoreHash int, provide
 	return b, diags
 }
 
-// isProviderReattached determines if a given provider is being supplied to Terraform via the TF_REATTACH_PROVIDERS
-// environment variable.
-func isProviderReattached(provider addrs.Provider) (bool, error) {
-	in := os.Getenv("TF_REATTACH_PROVIDERS")
-	if in != "" {
-		var m map[string]any
-		err := json.Unmarshal([]byte(in), &m)
-		if err != nil {
-			return false, fmt.Errorf("Invalid format for TF_REATTACH_PROVIDERS: %w", err)
-		}
-		for p := range m {
-			a, diags := addrs.ParseProviderSourceString(p)
-			if diags.HasErrors() {
-				return false, fmt.Errorf("Error parsing %q as a provider address: %w", a, diags.Err())
-			}
-			if a.Equals(provider) {
-				return true, nil
-			}
-		}
-	}
-	return false, nil
-}
-
 // createDefaultWorkspace receives a backend made using a pluggable state store, and details about that store's config,
 // and persists an empty state file in the default workspace. By creating this artifact we ensure that the default
 // workspace is created and usable by Terraform in later operations.

internal/command/workdir/backend_state_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	tfaddr "github.com/hashicorp/terraform-registry-address"
 	"github.com/hashicorp/terraform/version"
 )
 
@@ -160,9 +161,12 @@ func TestParseBackendStateFile(t *testing.T) {
 }
 
 func TestEncodeBackendStateFile(t *testing.T) {
+	noVersionData := ""
+
 	tfVersion := version.Version
 	tests := map[string]struct {
 		Input   *BackendStateFile
+		Envs    map[string]string
 		Want    []byte
 		WantErr string
 	}{
@@ -177,11 +181,58 @@ func TestEncodeBackendStateFile(t *testing.T) {
 			},
 			Want: []byte("{\n  \"version\": 3,\n  \"terraform_version\": \"" + tfVersion + "\",\n  \"state_store\": {\n    \"type\": \"foobar_baz\",\n    \"provider\": {\n      \"version\": \"1.2.3\",\n      \"source\": \"registry.terraform.io/my-org/foobar\",\n      \"config\": {\n        \"foo\": \"bar\"\n      },\n      \"hash\": 12345\n    },\n    \"config\": {\n      \"foo\": \"bar\"\n    },\n    \"hash\": 123\n  }\n}"),
 		},
-		"it returns an error when neither backend nor state_store config state are present": {
+		"it's valid to record no version data when a builtin provider used for state store": {
+			Input: &BackendStateFile{
+				StateStore: &StateStoreConfigState{
+					Type:      "foobar_baz",
+					Provider:  getTestProviderState(t, noVersionData, string(tfaddr.BuiltInProviderHost), string(tfaddr.BuiltInProviderNamespace), "foobar", `{"foo": "bar"}`),
+					ConfigRaw: json.RawMessage([]byte(`{"foo":"bar"}`)),
+					Hash:      123,
+				},
+			},
+			Want: []byte("{\n  \"version\": 3,\n  \"terraform_version\": \"" + tfVersion + "\",\n  \"state_store\": {\n    \"type\": \"foobar_baz\",\n    \"provider\": {\n      \"version\": null,\n      \"source\": \"terraform.io/builtin/foobar\",\n      \"config\": {\n        \"foo\": \"bar\"\n      },\n      \"hash\": 12345\n    },\n    \"config\": {\n      \"foo\": \"bar\"\n    },\n    \"hash\": 123\n  }\n}"),
+		},
+		"it's valid to record no version data when a re-attached provider used for state store": {
+			Input: &BackendStateFile{
+				StateStore: &StateStoreConfigState{
+					Type:      "foobar_baz",
+					Provider:  getTestProviderState(t, noVersionData, "registry.terraform.io", "hashicorp", "foobar", `{"foo": "bar"}`),
+					ConfigRaw: json.RawMessage([]byte(`{"foo":"bar"}`)),
+					Hash:      123,
+				},
+			},
+			Envs: map[string]string{
+				"TF_REATTACH_PROVIDERS": `{
+				"foobar": {
+					"Protocol": "grpc",
+					"ProtocolVersion": 6,
+					"Pid": 12345,
+					"Test": true,
+					"Addr": {
+						"Network": "unix",
+						"String":"/var/folders/xx/abcde12345/T/plugin12345"
+					}
+				}
+			}`,
+			},
+			Want: []byte("{\n  \"version\": 3,\n  \"terraform_version\": \"" + tfVersion + "\",\n  \"state_store\": {\n    \"type\": \"foobar_baz\",\n    \"provider\": {\n      \"version\": null,\n      \"source\": \"registry.terraform.io/hashicorp/foobar\",\n      \"config\": {\n        \"foo\": \"bar\"\n      },\n      \"hash\": 12345\n    },\n    \"config\": {\n      \"foo\": \"bar\"\n    },\n    \"hash\": 123\n  }\n}"),
+		},
+		"error when neither backend nor state_store config state are present": {
 			Input: &BackendStateFile{},
 			Want:  []byte("{\n  \"version\": 3,\n  \"terraform_version\": \"" + tfVersion + "\"\n}"),
 		},
-		"it returns an error when the provider source's hostname is missing": {
+		"error when the provider is neither builtin nor reattached and the provider version is missing": {
+			Input: &BackendStateFile{
+				StateStore: &StateStoreConfigState{
+					Type:      "foobar_baz",
+					Provider:  getTestProviderState(t, noVersionData, "registry.terraform.io", "my-org", "foobar", ""),
+					ConfigRaw: json.RawMessage([]byte(`{"foo":"bar"}`)),
+					Hash:      123,
+				},
+			},
+			WantErr: `state store is not valid: provider version data is missing`,
+		},
+		"error when the provider source's hostname is missing": {
 			Input: &BackendStateFile{
 				StateStore: &StateStoreConfigState{
 					Type:      "foobar_baz",
@@ -192,7 +243,7 @@ func TestEncodeBackendStateFile(t *testing.T) {
 			},
 			WantErr: `state store is not valid: Unknown hostname: Expected hostname in the provider address to be set`,
 		},
-		"it returns an error when the provider source's hostname and namespace are missing ": {
+		"error when the provider source's hostname and namespace are missing ": {
 			Input: &BackendStateFile{
 				StateStore: &StateStoreConfigState{
 					Type:      "foobar_baz",
@@ -203,7 +254,7 @@ func TestEncodeBackendStateFile(t *testing.T) {
 			},
 			WantErr: `state store is not valid: Unknown hostname: Expected hostname in the provider address to be set`,
 		},
-		"it returns an error when the provider source is completely missing ": {
+		"error when the provider source is completely missing ": {
 			Input: &BackendStateFile{
 				StateStore: &StateStoreConfigState{
 					Type:      "foobar_baz",
@@ -214,7 +265,7 @@ func TestEncodeBackendStateFile(t *testing.T) {
 			},
 			WantErr: `state store is not valid: Empty provider address: Expected address composed of hostname, provider namespace and name`,
 		},
-		"it returns an error when both backend and state_store config state are present": {
+		"error when both backend and state_store config state are present": {
 			Input: &BackendStateFile{
 				Backend: &BackendConfigState{
 					Type:      "foobar",
@@ -234,6 +285,11 @@ func TestEncodeBackendStateFile(t *testing.T) {
 
 	for name, test := range tests {
 		t.Run(name, func(t *testing.T) {
+			// Some test cases depend on ENVs, not all
+			for k, v := range test.Envs {
+				t.Setenv(k, v)
+			}
+
 			got, err := EncodeBackendStateFile(test.Input)
 
 			if test.WantErr != "" {

internal/command/workdir/statestore_config_state.go

@@ -7,10 +7,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 
 	version "github.com/hashicorp/go-version"
 	tfaddr "github.com/hashicorp/terraform-registry-address"
 	"github.com/hashicorp/terraform/internal/configs/configschema"
+	"github.com/hashicorp/terraform/internal/getproviders/reattach"
 	"github.com/hashicorp/terraform/internal/plans"
 	"github.com/zclconf/go-cty/cty"
 	ctyjson "github.com/zclconf/go-cty/cty/json"
@@ -40,13 +42,13 @@ func (s *StateStoreConfigState) Validate() error {
 
 	// Are any bits of data totally missing?
 	if s.Empty() {
-		return fmt.Errorf("state store is not valid: data is empty")
+		return fmt.Errorf("attempted to encode a malformed backend state file; data is empty")
 	}
-	if s.Provider == nil {
-		return fmt.Errorf("state store is not valid: provider data is missing")
+	if s.Type == "" {
+		return fmt.Errorf("attempted to encode a malformed backend state file; state store type is missing")
 	}
-	if s.Provider.Version == nil {
-		return fmt.Errorf("state store is not valid: version data is missing")
+	if s.Provider == nil {
+		return fmt.Errorf("attempted to encode a malformed backend state file; provider data is missing")
 	}
 	if s.ConfigRaw == nil {
 		return fmt.Errorf("attempted to encode a malformed backend state file; state_store configuration data is missing")
@@ -58,6 +60,18 @@ func (s *StateStoreConfigState) Validate() error {
 		return fmt.Errorf("state store is not valid: %w", err)
 	}
 
+	// Version information is required if the provider isn't builtin or unmanaged by Terraform
+	isReattached, err := reattach.IsProviderReattached(*s.Provider.Source, os.Getenv("TF_REATTACH_PROVIDERS"))
+	if err != nil {
+		return fmt.Errorf("error determining if state storage provider is reattached: %w", err)
+	}
+	if (s.Provider.Source.Hostname != tfaddr.BuiltInProviderHost) &&
+		!isReattached {
+		if s.Provider.Version == nil {
+			return fmt.Errorf("state store is not valid: provider version data is missing")
+		}
+	}
+
 	return nil
 }
 

internal/command/workdir/testing.go

@@ -17,9 +17,16 @@ import (
 func getTestProviderState(t *testing.T, semVer, hostname, namespace, typeName, config string) *ProviderConfigState {
 	t.Helper()
 
-	ver, err := version.NewSemver(semVer)
-	if err != nil {
-		t.Fatalf("test setup failed when creating version.Version: %s", err)
+	var ver *version.Version
+	if semVer == "" {
+		// Allow passing no version in; leave ver nil
+		ver = nil
+	} else {
+		var err error
+		ver, err = version.NewSemver(semVer)
+		if err != nil {
+			t.Fatalf("test setup failed when creating version.Version: %s", err)
+		}
 	}
 
 	return &ProviderConfigState{

internal/getproviders/reattach/reattach.go

@@ -9,6 +9,7 @@ import (
 	"net"
 
 	"github.com/hashicorp/go-plugin"
+	tfaddr "github.com/hashicorp/terraform-registry-address"
 	"github.com/hashicorp/terraform/internal/addrs"
 )
 
@@ -88,7 +89,7 @@ func ParseReattachProviders(in string) (map[addrs.Provider]*plugin.ReattachConfi
 // environment variable.
 //
 // Calling code is expected to pass in a provider address and the value of os.Getenv("TF_REATTACH_PROVIDERS")
-func IsProviderReattached(provider addrs.Provider, in string) (bool, error) {
+func IsProviderReattached(provider tfaddr.Provider, in string) (bool, error) {
 	providers, err := ParseReattachProviders(in)
 	if err != nil {
 		return false, err