diff --git a/internal/command/e2etest/pluggable_state_store_test.go b/internal/command/e2etest/pluggable_state_store_test.go index 358c4b26d9f4..5540904b4229 100644 --- a/internal/command/e2etest/pluggable_state_store_test.go +++ b/internal/command/e2etest/pluggable_state_store_test.go @@ -4,6 +4,7 @@ package e2etest import ( + "bytes" "fmt" "os" "path" @@ -12,8 +13,12 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform/internal/addrs" "github.com/hashicorp/terraform/internal/e2e" "github.com/hashicorp/terraform/internal/getproviders" + "github.com/hashicorp/terraform/internal/states" + "github.com/hashicorp/terraform/internal/states/statefile" ) // Tests using `terraform workspace` commands in combination with pluggable state storage. @@ -350,3 +355,153 @@ greeting = "hello world" // TODO(SarahFrench/radeksimko): Show plan file: terraform show } + +// Tests using the `terraform provider` subcommands in combination with pluggable state storage: +// > `terraform providers` +// > `terraform providers schema` +// +// Commands `terraform providers locks` and `terraform providers mirror` aren't tested as they +// don't interact with the backend. +// +// The test `TestProvidersSchema` has test coverage showing that state store schemas are present +// in the command's outputs. _This_ test is intended to assert that the command is able to read and use +// state via a state store ok, and is able to detect providers required only by the state. +func TestPrimary_stateStore_providerCmds(t *testing.T) { + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provider executable") + } + + t.Setenv(e2e.TestExperimentFlag, "true") + terraformBin := e2e.GoBuild("github.com/hashicorp/terraform", "terraform") + fixturePath := filepath.Join("testdata", "full-workflow-with-state-store-fs") + tf := e2e.NewBinary(t, terraformBin, fixturePath) + workspaceDirName := "states" // See workspace_dir value in the configuration + + // Add a state file describing a resource from the simple (v5) provider, so + // we can test that the state is read and used to get all the provider schemas + fakeState := states.BuildState(func(s *states.SyncState) { + s.SetResourceInstanceCurrent( + addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "simple_resource", + Name: "foo", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + &states.ResourceInstanceObjectSrc{ + AttrsJSON: []byte(`{"id":"bar"}`), + Status: states.ObjectReady, + }, + addrs.AbsProviderConfig{ + Provider: addrs.NewDefaultProvider("simple"), + Module: addrs.RootModule, + }, + ) + }) + fakeStateFile := &statefile.File{ + Lineage: "boop", + Serial: 4, + TerraformVersion: version.Must(version.NewVersion("1.0.0")), + State: fakeState, + } + var fakeStateBuf bytes.Buffer + err := statefile.WriteForTest(fakeStateFile, &fakeStateBuf) + if err != nil { + t.Error(err) + } + fakeStateBytes := fakeStateBuf.Bytes() + + if err := os.MkdirAll(tf.Path(workspaceDirName, "default"), os.ModePerm); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(tf.Path(workspaceDirName, "default", "terraform.tfstate"), fakeStateBytes, 0644); err != nil { + t.Fatal(err) + } + + // In order to test integration with PSS we need a provider plugin implementing a state store. + // Here will build the simple6 (built with protocol v6) provider, which will be used for PSS. + // The simple (v5) provider is also built, as that provider will be present in the state and therefore + // needed for creating the schema output. + simple6Provider := filepath.Join(tf.WorkDir(), "terraform-provider-simple6") + simple6ProviderExe := e2e.GoBuild("github.com/hashicorp/terraform/internal/provider-simple-v6/main", simple6Provider) + + simpleProvider := filepath.Join(tf.WorkDir(), "terraform-provider-simple") + simpleProviderExe := e2e.GoBuild("github.com/hashicorp/terraform/internal/provider-simple/main", simpleProvider) + + // Move the provider binaries into a directory that we will point terraform + // to using the -plugin-dir cli flag. + platform := getproviders.CurrentPlatform.String() + fsMirrorPathV6 := "cache/registry.terraform.io/hashicorp/simple6/0.0.1/" + if err := os.MkdirAll(tf.Path(fsMirrorPathV6, platform), os.ModePerm); err != nil { + t.Fatal(err) + } + if err := os.Rename(simple6ProviderExe, tf.Path(fsMirrorPathV6, platform, "terraform-provider-simple6")); err != nil { + t.Fatal(err) + } + + fsMirrorPathV5 := "cache/registry.terraform.io/hashicorp/simple/0.0.1/" + if err := os.MkdirAll(tf.Path(fsMirrorPathV5, platform), os.ModePerm); err != nil { + t.Fatal(err) + } + if err := os.Rename(simpleProviderExe, tf.Path(fsMirrorPathV5, platform, "terraform-provider-simple")); err != nil { + t.Fatal(err) + } + + //// Init + _, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir=cache", "-no-color") + if err != nil { + t.Fatalf("unexpected error: %s\nstderr:\n%s", err, stderr) + } + fi, err := os.Stat(path.Join(tf.WorkDir(), workspaceDirName, "default", "terraform.tfstate")) + if err != nil { + t.Fatalf("failed to open default workspace's state file: %s", err) + } + if fi.Size() == 0 { + t.Fatal("default workspace's state file should not have size 0 bytes") + } + + //// Providers: `terraform providers` + stdout, stderr, err := tf.Run("providers", "-no-color") + if err != nil { + t.Fatalf("unexpected error: %s\nstderr:\n%s", err, stderr) + } + + // We expect the command to be able to use the state store to + // detect providers that come from only the state. + expectedMsgs := []string{ + "Providers required by configuration:", + "provider[registry.terraform.io/hashicorp/simple6]", + "provider[terraform.io/builtin/terraform]", + "Providers required by state:", + "provider[registry.terraform.io/hashicorp/simple]", + } + for _, msg := range expectedMsgs { + if !strings.Contains(stdout, msg) { + t.Errorf("unexpected output, expected %q, but got:\n%s", msg, stdout) + } + } + + //// Provider schemas: `terraform providers schema` + stdout, stderr, err = tf.Run("providers", "schema", "-json", "-no-color") + if err != nil { + t.Fatalf("unexpected error: %s\nstderr:\n%s", err, stderr) + } + + expectedMsgs = []string{ + `"registry.terraform.io/hashicorp/simple6"`, // provider used for PSS + `"terraform.io/builtin/terraform"`, // provider used for resources + `"registry.terraform.io/hashicorp/simple"`, // provider present only in the state + } + for _, msg := range expectedMsgs { + if !strings.Contains(stdout, msg) { + t.Errorf("unexpected output, expected %q, but got:\n%s", msg, stdout) + } + } + + // More thorough checking of the JSON output is in `TestProvidersSchema`. + // This test just asserts that `terraform providers schema` can read state + // via the state store, and therefore detects all 3 providers needed for the output. +} diff --git a/internal/command/e2etest/providers_schema_test.go b/internal/command/e2etest/providers_schema_test.go index d729afdbacd6..3c7452a957d2 100644 --- a/internal/command/e2etest/providers_schema_test.go +++ b/internal/command/e2etest/providers_schema_test.go @@ -269,6 +269,33 @@ func TestProvidersSchema(t *testing.T) { } } } + }, + "state_store_schemas" : { + "simple6_fs": { + "version":0, + "block": { + "attributes": { + "workspace_dir": { + "type":"string", + "description":"The directory where state files will be created. When unset the value will default to terraform.tfstate.d","description_kind":"plain","optional":true} + }, + "description_kind":"plain" + } + }, + "simple6_inmem": { + "version": 0, + "block": { + "attributes": { + "lock_id": { + "type": "string", + "description": "initializes the state in a locked configuration", + "description_kind": "plain", + "optional": true + } + }, + "description_kind":"plain" + } + } } } } diff --git a/internal/command/jsonprovider/provider.go b/internal/command/jsonprovider/provider.go index 10fdcf7831f4..148b91f2374b 100644 --- a/internal/command/jsonprovider/provider.go +++ b/internal/command/jsonprovider/provider.go @@ -31,6 +31,7 @@ type Provider struct { Functions map[string]*jsonfunction.FunctionSignature `json:"functions,omitempty"` ResourceIdentitySchemas map[string]*IdentitySchema `json:"resource_identity_schemas,omitempty"` ActionSchemas map[string]*ActionSchema `json:"action_schemas,omitempty"` + StateStoreSchemas map[string]*Schema `json:"state_store_schemas,omitempty"` } func newProviders() *Providers { @@ -69,6 +70,7 @@ func marshalProvider(tps providers.ProviderSchema) *Provider { Functions: jsonfunction.MarshalProviderFunctions(tps.Functions), ResourceIdentitySchemas: marshalIdentitySchemas(tps.ResourceTypes), ActionSchemas: marshalActionSchemas(tps.Actions), + StateStoreSchemas: marshalSchemas(tps.StateStores), } // List resource schemas are nested under a "config" block, so we need to diff --git a/internal/command/jsonprovider/provider_test.go b/internal/command/jsonprovider/provider_test.go index bb9ca97486b1..ce84210b6240 100644 --- a/internal/command/jsonprovider/provider_test.go +++ b/internal/command/jsonprovider/provider_test.go @@ -33,6 +33,7 @@ func TestMarshalProvider(t *testing.T) { ResourceIdentitySchemas: map[string]*IdentitySchema{}, ListResourceSchemas: map[string]*Schema{}, ActionSchemas: map[string]*ActionSchema{}, + StateStoreSchemas: map[string]*Schema{}, }, }, { @@ -250,6 +251,7 @@ func TestMarshalProvider(t *testing.T) { }, }, }, + StateStoreSchemas: map[string]*Schema{}, }, }, } diff --git a/internal/command/providers_schema_test.go b/internal/command/providers_schema_test.go index fdfc3e1747ce..87b0cd5900c9 100644 --- a/internal/command/providers_schema_test.go +++ b/internal/command/providers_schema_test.go @@ -4,20 +4,25 @@ package command import ( + "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" "os" "path/filepath" + "strings" "testing" "github.com/google/go-cmp/cmp" "github.com/hashicorp/cli" "github.com/zclconf/go-cty/cty" + "github.com/hashicorp/terraform/internal/addrs" "github.com/hashicorp/terraform/internal/configs/configschema" "github.com/hashicorp/terraform/internal/providers" testing_provider "github.com/hashicorp/terraform/internal/providers/testing" + "github.com/hashicorp/terraform/internal/states" + "github.com/hashicorp/terraform/internal/states/statefile" ) func TestProvidersSchema_error(t *testing.T) { @@ -37,7 +42,7 @@ func TestProvidersSchema_error(t *testing.T) { func TestProvidersSchema_output(t *testing.T) { fixtureDir := "testdata/providers-schema" - testDirs, err := ioutil.ReadDir(fixtureDir) + testDirs, err := os.ReadDir(fixtureDir) if err != nil { t.Fatal(err) } @@ -90,7 +95,7 @@ func TestProvidersSchema_output(t *testing.T) { t.Fatalf("err: %s", err) } defer wantFile.Close() - byteValue, err := ioutil.ReadAll(wantFile) + byteValue, err := io.ReadAll(wantFile) if err != nil { t.Fatalf("err: %s", err) } @@ -103,6 +108,109 @@ func TestProvidersSchema_output(t *testing.T) { } } +func TestProvidersSchema_output_withStateStore(t *testing.T) { + // State with a 'baz' provider not in the config + originalState := states.BuildState(func(s *states.SyncState) { + s.SetResourceInstanceCurrent( + addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "baz_instance", + Name: "foo", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + &states.ResourceInstanceObjectSrc{ + AttrsJSON: []byte(`{"id":"bar"}`), + Status: states.ObjectReady, + }, + addrs.AbsProviderConfig{ + Provider: addrs.NewDefaultProvider("baz"), + Module: addrs.RootModule, + }, + ) + }) + + // Create a temporary working directory that includes config using + // a state store in the `test` provider + td := t.TempDir() + testCopyDir(t, testFixturePath("provider-schemas-state-store"), td) + t.Chdir(td) + + // Get bytes describing the state + var stateBuf bytes.Buffer + if err := statefile.Write(statefile.New(originalState, "", 1), &stateBuf); err != nil { + t.Fatalf("error during test setup: %s", err) + } + + // Create a mock that contains a persisted "default" state that uses the bytes from above. + mockProvider := mockPluggableStateStorageProvider() + mockProvider.MockStates = map[string]interface{}{ + "default": stateBuf.Bytes(), + } + mockProviderAddressTest := addrs.NewDefaultProvider("test") + + // Mock for the provider in the state + mockProviderAddressBaz := addrs.NewDefaultProvider("baz") + + ui := new(cli.MockUi) + c := &ProvidersSchemaCommand{ + Meta: Meta{ + Ui: ui, + AllowExperimentalFeatures: true, + testingOverrides: &testingOverrides{ + Providers: map[addrs.Provider]providers.Factory{ + mockProviderAddressTest: providers.FactoryFixed(mockProvider), + mockProviderAddressBaz: providers.FactoryFixed(mockProvider), + }, + }, + }, + } + + args := []string{"-json"} + if code := c.Run(args); code != 0 { + t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String()) + } + + // Does the output mention the 2 providers, and the name of the state store? + wantOutput := []string{ + mockProviderAddressBaz.String(), // provider from state + mockProviderAddressTest.String(), // provider from config + "test_store", // the name of the state store implemented in the provider + } + + output := ui.OutputWriter.String() + for _, want := range wantOutput { + if !strings.Contains(output, want) { + t.Errorf("output missing %s:\n%s", want, output) + } + } + + // Does the output match the full expected schema? + var got, want providerSchemas + + gotString := ui.OutputWriter.String() + err := json.Unmarshal([]byte(gotString), &got) + if err != nil { + t.Fatal(err) + } + + wantFile, err := os.Open("output.json") + if err != nil { + t.Fatalf("err: %s", err) + } + defer wantFile.Close() + byteValue, err := io.ReadAll(wantFile) + if err != nil { + t.Fatalf("err: %s", err) + } + err = json.Unmarshal([]byte(byteValue), &want) + if err != nil { + t.Fatal(err) + } + + if !cmp.Equal(got, want) { + t.Fatalf("wrong result:\n %v\n", cmp.Diff(got, want)) + } +} + type providerSchemas struct { FormatVersion string `json:"format_version"` Schemas map[string]providerSchema `json:"provider_schemas"` @@ -112,6 +220,7 @@ type providerSchema struct { Provider interface{} `json:"provider,omitempty"` ResourceSchemas map[string]interface{} `json:"resource_schemas,omitempty"` DataSourceSchemas map[string]interface{} `json:"data_source_schemas,omitempty"` + StateStoreSchemas map[string]interface{} `json:"state_store_schemas,omitempty"` } // testProvider returns a mock provider that is configured for basic diff --git a/internal/command/providers_test.go b/internal/command/providers_test.go index 596a66b77b1f..b2755894535c 100644 --- a/internal/command/providers_test.go +++ b/internal/command/providers_test.go @@ -4,11 +4,16 @@ package command import ( + "bytes" "os" "strings" "testing" "github.com/hashicorp/cli" + "github.com/hashicorp/terraform/internal/addrs" + "github.com/hashicorp/terraform/internal/providers" + "github.com/hashicorp/terraform/internal/states" + "github.com/hashicorp/terraform/internal/states/statefile" ) func TestProviders(t *testing.T) { @@ -203,3 +208,74 @@ func TestProviders_tests(t *testing.T) { } } } + +func TestProviders_state_withStateStore(t *testing.T) { + // State with a 'baz' provider not in the config + originalState := states.BuildState(func(s *states.SyncState) { + s.SetResourceInstanceCurrent( + addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "baz_instance", + Name: "foo", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + &states.ResourceInstanceObjectSrc{ + AttrsJSON: []byte(`{"id":"bar"}`), + Status: states.ObjectReady, + }, + addrs.AbsProviderConfig{ + Provider: addrs.NewDefaultProvider("baz"), + Module: addrs.RootModule, + }, + ) + }) + + // Create a temporary working directory that is empty + td := t.TempDir() + testCopyDir(t, testFixturePath("state-store-unchanged"), td) + t.Chdir(td) + + // Get bytes describing the state + var stateBuf bytes.Buffer + if err := statefile.Write(statefile.New(originalState, "", 1), &stateBuf); err != nil { + t.Fatalf("error during test setup: %s", err) + } + + // Create a mock that contains a persisted "default" state that uses the bytes from above. + mockProvider := mockPluggableStateStorageProvider() + mockProvider.MockStates = map[string]interface{}{ + "default": stateBuf.Bytes(), + } + mockProviderAddress := addrs.NewDefaultProvider("test") + + ui := new(cli.MockUi) + c := &ProvidersCommand{ + Meta: Meta{ + Ui: ui, + AllowExperimentalFeatures: true, + testingOverrides: &testingOverrides{ + Providers: map[addrs.Provider]providers.Factory{ + mockProviderAddress: providers.FactoryFixed(mockProvider), + }, + }, + }, + } + + args := []string{} + if code := c.Run(args); code != 0 { + t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String()) + } + + wantOutput := []string{ + "Providers required by configuration:", + "└── provider[registry.terraform.io/hashicorp/test] 1.2.3", + "Providers required by state:", + "provider[registry.terraform.io/hashicorp/baz]", + } + + output := ui.OutputWriter.String() + for _, want := range wantOutput { + if !strings.Contains(output, want) { + t.Errorf("output missing %s:\n%s", want, output) + } + } +} diff --git a/internal/command/testdata/provider-schemas-state-store/.terraform.lock.hcl b/internal/command/testdata/provider-schemas-state-store/.terraform.lock.hcl new file mode 100644 index 000000000000..e5c03757a7fa --- /dev/null +++ b/internal/command/testdata/provider-schemas-state-store/.terraform.lock.hcl @@ -0,0 +1,6 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/test" { + version = "1.2.3" +} diff --git a/internal/command/testdata/provider-schemas-state-store/.terraform/terraform.tfstate b/internal/command/testdata/provider-schemas-state-store/.terraform/terraform.tfstate new file mode 100644 index 000000000000..4f96aa73ee7d --- /dev/null +++ b/internal/command/testdata/provider-schemas-state-store/.terraform/terraform.tfstate @@ -0,0 +1,19 @@ +{ + "version": 3, + "serial": 0, + "lineage": "666f9301-7e65-4b19-ae23-71184bb19b03", + "state_store": { + "type": "test_store", + "config": { + "value": "foobar" + }, + "provider": { + "version": "1.2.3", + "source": "registry.terraform.io/hashicorp/test", + "config": { + "region": null + } + }, + "hash": 4158988729 + } +} \ No newline at end of file diff --git a/internal/command/testdata/provider-schemas-state-store/README.md b/internal/command/testdata/provider-schemas-state-store/README.md new file mode 100644 index 000000000000..7b6cb99a79e9 --- /dev/null +++ b/internal/command/testdata/provider-schemas-state-store/README.md @@ -0,0 +1,2 @@ +This test fixture is a working directory that contains a state_store block that matches the backend state file and dependency lock file. +Any test using this fixture will need to set up a mock provider that describes a state store in its schema that matches this config. \ No newline at end of file diff --git a/internal/command/testdata/provider-schemas-state-store/main.tf b/internal/command/testdata/provider-schemas-state-store/main.tf new file mode 100644 index 000000000000..df1eaa76b650 --- /dev/null +++ b/internal/command/testdata/provider-schemas-state-store/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + test = { + source = "hashicorp/test" + version = "1.2.3" + } + } + state_store "test_store" { + provider "test" {} + + value = "foobar" # matches backend state file + } +} diff --git a/internal/command/testdata/provider-schemas-state-store/output.json b/internal/command/testdata/provider-schemas-state-store/output.json new file mode 100644 index 000000000000..39957788eea3 --- /dev/null +++ b/internal/command/testdata/provider-schemas-state-store/output.json @@ -0,0 +1,105 @@ +{ + "format_version": "1.0", + "provider_schemas": { + "registry.terraform.io/hashicorp/baz": { + "provider": { + "version": 0, + "block": { + "attributes": { + "region": { + "description_kind": "plain", + "optional": true, + "type": "string" + } + }, + "description_kind": "plain" + } + }, + "resource_schemas": { + "test_instance": { + "version": 0, + "block": { + "attributes": { + "id": { + "type": "string", + "computed": true, + "description_kind": "plain" + }, + "input": { + "description_kind": "plain", + "optional": true, + "type": "string" + } + }, + "description_kind": "plain" + } + } + }, + "state_store_schemas": { + "test_store": { + "version": 0, + "block": { + "attributes": { + "value": { + "description_kind": "plain", + "required": true, + "type": "string" + } + }, + "description_kind": "plain" + } + } + } + }, + "registry.terraform.io/hashicorp/test": { + "provider": { + "version": 0, + "block": { + "attributes": { + "region": { + "description_kind": "plain", + "optional": true, + "type": "string" + } + }, + "description_kind": "plain" + } + }, + "resource_schemas": { + "test_instance": { + "version": 0, + "block": { + "attributes": { + "id": { + "type": "string", + "computed": true, + "description_kind": "plain" + }, + "input": { + "description_kind": "plain", + "optional": true, + "type": "string" + } + }, + "description_kind": "plain" + } + } + }, + "state_store_schemas": { + "test_store": { + "version": 0, + "block": { + "attributes": { + "value": { + "description_kind": "plain", + "required": true, + "type": "string" + } + }, + "description_kind": "plain" + } + } + } + } + } +} \ No newline at end of file