Convert DCL resources to handwritten

GNUmakefile

@@ -4,41 +4,16 @@
 default: build
 
 # mm setup
-ifeq ($(ENGINE),tpgtools)
-  # we specify the product to one that doesn't
-  # exist so exclusively build base tpgtools implementation
-  mmv1_compile=-p does-not-exist
-else ifneq ($(PRODUCT),)
+ifneq ($(PRODUCT),)
   mmv1_compile=--product $(PRODUCT)
 endif
 
-# tpgtools setup
-ifeq ($(ENGINE),mmv1)
-  # we specify the product to one that doesn't
-  # exist so exclusively build base mmv1 implementation
-	tpgtools_compile = --service does-not-exist
-else ifneq ($(PRODUCT),)
-  tpgtools_compile = --service $(PRODUCT)
-else
-  tpgtools_compile =
-endif
-
 ifneq ($(RESOURCE),)
   mmv1_compile += --resource $(RESOURCE)
-  tpgtools_compile += --resource $(RESOURCE)
 endif
 
 ifneq ($(OVERRIDES),)
   mmv1_compile += --overrides $(OVERRIDES)
-  tpgtools_compile += --overrides $(OVERRIDES)/tpgtools/overrides --path $(OVERRIDES)/tpgtools/api
-  serialize_compile = --overrides $(OVERRIDES)/tpgtools/overrides --path $(OVERRIDES)/tpgtools/api
-else
-  tpgtools_compile += --path "api" --overrides "overrides"
-  serialize_compile = --path "api" --overrides "overrides"
-endif
-
-ifneq ($(VERBOSE),)
-  tpgtools_compile += --logtostderr=1 --stderrthreshold=2
 endif
 
 UNAME := $(shell uname)
@@ -50,18 +25,14 @@ else
 SED_I := -i '' -E
 endif
 
-ifeq ($(FORCE_DCL),)
-  FORCE_DCL=latest
-endif
-
 SHOULD_SKIP_CLEAN := false # Default: do not skip
 ifneq ($(SKIP_CLEAN),)
   ifneq ($(SKIP_CLEAN),false)
     SHOULD_SKIP_CLEAN := true
   endif
 endif
 
-terraform build provider: validate_environment clean-provider mmv1 tpgtools
+terraform build provider: validate_environment clean-provider mmv1
 	@echo "Provider generation process finished for $(VERSION) in $(OUTPUT_PATH)"
 
 
@@ -75,12 +46,6 @@ mmv1:
 			go run . --output $(OUTPUT_PATH) --version $(VERSION) $(mmv1_compile); \
 		fi
 
-tpgtools: serialize
-	@echo "Executing tpgtools build for $(OUTPUT_PATH)";
-	@cd tpgtools;\
-		go run . --output $(OUTPUT_PATH) --version $(VERSION) $(tpgtools_compile); \
-		rm serialization.go
-
 clean-provider: check_safe_build
 	@if [ -n "$(PRODUCT)" ]; then \
 		printf "\n\e[1;33mWARNING:\e[0m Skipping clean-provider step because PRODUCT ('$(PRODUCT)') is set.\n"; \
@@ -136,24 +101,6 @@ test:
 	cd mmv1; \
 		go test ./...
 
-serialize:
-	cd tpgtools;\
-		cp -f serialization.go.base serialization.go &&\
-		go run . $(serialize_compile) --mode "serialization" > temp.serial &&\
-		mv -f temp.serial serialization.go
-
-upgrade-dcl:
-	make serialize
-	cd tpgtools && \
-		go mod edit -dropreplace=github.com/GoogleCloudPlatform/declarative-resource-client-library &&\
-		go mod edit -require=github.com/GoogleCloudPlatform/declarative-resource-client-library@$(FORCE_DCL) &&\
-		go mod tidy;\
-		MOD_LINE=$$(grep declarative-resource-client-library go.mod);\
-		SUM_LINE=$$(grep declarative-resource-client-library go.sum);\
-	cd ../mmv1/third_party/terraform && \
-		sed ${SED_I} "s!.*declarative-resource-client-library.*!$$MOD_LINE!" go.mod; echo "$$SUM_LINE" >> go.sum
-
-
 validate_environment: check_parameters check_safe_build
 
 check_parameters:
@@ -175,4 +122,4 @@ check_safe_build:
 doctor:
 	./scripts/doctor
 
-.PHONY: mmv1 tpgtools test clean-provider validate_environment serialize doctor
+.PHONY: mmv1 test clean-provider validate_environment doctor

docs/content/_index.md

@@ -35,11 +35,10 @@ detection.
 
 ### Resource types
 
-There are three types of resources supported by Magic Modules:
+There are two types of resources supported by Magic Modules:
 
 + MMv1
 + Handwritten
-+ DCL/tpgtools
 
 The following sections describe these tools in detail.
 
@@ -96,17 +95,6 @@ In the providers, handwritten resources and datasources are stored in `PROVIDER/
 is `google` or `google-beta`, `SERVICE` is the service name, and `FILENAME` is the name of the handwritten file in magic-modules.
 Handwritten files do not have an `AUTO GENERATED CODE` header.
 
-#### DCL aka tpgtools (maintenance mode)
-
-DCL / tpgtools is similar to MMv1; however, it is in maintenance mode, which means that new resources using the DCL are not being added.
-
-DCL-based files start with the following header:
-
-```
-***     AUTO GENERATED CODE    ***    Type: DCL     ***
-```
-
-
 ## Other Resources
 
 + [Extending Terraform](https://www.terraform.io/plugin)

docs/content/code-review/release-notes.md

@@ -98,10 +98,10 @@ For each release note block, choose an appropriate type from the following list:
 Do | Don't
 -- | -----
 Use past tense to describe the end state after the change is released. Start with a verb. For example, "added...", "fixed...", or "resolved...". You can use future tense to describe future changes, such as saying that a deprecated field will be removed in a future version. | Don't use present or future tense to describe changes that are included in the pull request.
-Write user-focused release notes. For example, reference specific impacted terraform resource and field names, and discuss changes in behavior users will experience. | Avoid API field/resource/feature names. Avoid implementation details. Avoid language that requires understanding of provider internals. However, in case of substantial refactorings like API version changes or engine changes (tpgtools/DCL -> MMv1, handwritten <> MMv1) **do** cover the change so users can quickly identify the release if they are affected by the change.
+Write user-focused release notes. For example, reference specific impacted terraform resource and field names, and discuss changes in behavior users will experience. | Avoid API field/resource/feature names. Avoid implementation details. Avoid language that requires understanding of provider internals. However, in case of substantial refactorings like API version changes or engine changes (handwritten <> MMv1) **do** cover the change so users can quickly identify the release if they are affected by the change.
 Surround resource or field names with backticks. | Don't use resource or field names without punctuation or with other punctuation like quotation marks.
 Use impersonal third person. | Don't use "I", "you", etc.
-If the pull request impacts a specific product, begin your release note with that product name followed by a colon. Use lower case for the first letter after the colon. For example, `cloudrun: added...` For MMv1 resources, use the folder name that contains the yaml files as the product name; for handwritten or tpgtools resources, use the API subdomain; for broad cross-product changes, use `provider`. | Don't begin your release note with the full resource name. Don't add backticks around the product name. Don't capitalize the first letter after the colon.
+If the pull request impacts a specific product, begin your release note with that product name followed by a colon. Use lower case for the first letter after the colon. For example, `cloudrun: added...` For MMv1 resources, use the folder name that contains the yaml files as the product name; for handwritten resources, use the API subdomain; for broad cross-product changes, use `provider`. | Don't begin your release note with the full resource name. Don't add backticks around the product name. Don't capitalize the first letter after the colon.
 
 ### Examples
 

docs/content/code-review/review-pr.md

@@ -13,7 +13,6 @@ This page provides guidelines for reviewing a Magic Modules pull request (PR).
 
 The following types of PRs may require additional scrutiny and/or multiple reviewers.
 
-- DCL to MMv1 migrations
 - Adding multi-actor fields (fields whose values can be altered as a side effect of changes made to a different resource)
 
 ## Review

docs/content/reference/make-commands.md

@@ -34,10 +34,9 @@ make provider VERSION=ga OUTPUT_PATH="$GOPATH/src/github.com/hashicorp/terraform
 
 - `OUTPUT_PATH`: Required. The location you are generating provider code into.
 - `VERSION`: Required. The version of the provider you are building into. Valid values are `ga` and `beta`.
-- `PRODUCT`: Limits generations to the specified folder within `mmv1/products` or `tpgtools/api`. Handwritten files from `mmv1/third_party/terraform` are always generated into the downstream regardless of this setting, so you can provide a non-existent product name to generate only handwritten code. Required if `RESOURCE` is specified. **Using `PRODUCT` skips the pre-generation cleanup step. This is considered advanced usage; recommend running a full, clean build (`make provider` without `PRODUCT`) beforehand if repositories may be out of sync.**
+- `PRODUCT`: Limits generations to the specified folder within `mmv1/products`. Handwritten files from `mmv1/third_party/terraform` are always generated into the downstream regardless of this setting, so you can provide a non-existent product name to generate only handwritten code. Required if `RESOURCE` is specified. **Using `PRODUCT` skips the pre-generation cleanup step. This is considered advanced usage; recommend running a full, clean build (`make provider` without `PRODUCT`) beforehand if repositories may be out of sync.**
 - `SKIP_CLEAN`: If set to `true`, skips the default pre-generation cleanup of `OUTPUT_PATH` during a full provider build. Has no effect if `PRODUCT` is specified (as cleanup is already skipped). Example: `make provider VERSION=ga OUTPUT_PATH=... SKIP_CLEAN=true`.
-- `RESOURCE`: Limits generation to the specified resource within a particular product. For `mmv1` resources, matches the resource's `name` field (set in its configuration file).For `tpgtools` resources, matches the terraform resource name.
-- `ENGINE`: Modifies `make provider` to only generate code using the specified engine. Valid values are `mmv1` or `tpgtools`. (Providing `tpgtools` will still generate any prerequisite mmv1 files required for tpgtools.)
+- `RESOURCE`: Limits generation to the specified resource within a particular product. For `mmv1` resources, matches the resource's `name` field (set in its configuration file).
 
 #### Cleaning up old files
 

docs/content/reference/metadata.md

@@ -9,7 +9,7 @@ This page documents the properties available in meta.yaml files. These files mak
 
 Meta.yaml files are auto-generated for MMv1 generated resources.
 
-DCL and Handwritten resources will have handwritten meta.yaml files in the appropriate service directory in [mmv1/third_party/terraform/services/](https://github.com/GoogleCloudPlatform/magic-modules/tree/main/mmv1/third_party/terraform/services). The file name will be `resource_PRODUCT_RESOURCE_meta.yaml(.tmpl)`. For example, `resource_compute_instance_meta.yaml.tmpl` Handwritten meta.yaml files with a `.tmpl` extension can use version guards (`{{- if ne $.TargetVersionName "ga" }}...{{- else}}...{{- end}}`) to exclude beta fields from the `google` provider.
+Handwritten resources will have handwritten meta.yaml files in the appropriate service directory in [mmv1/third_party/terraform/services/](https://github.com/GoogleCloudPlatform/magic-modules/tree/main/mmv1/third_party/terraform/services). The file name will be `resource_PRODUCT_RESOURCE_meta.yaml(.tmpl)`. For example, `resource_compute_instance_meta.yaml.tmpl` Handwritten meta.yaml files with a `.tmpl` extension can use version guards (`{{- if ne $.TargetVersionName "ga" }}...{{- else}}...{{- end}}`) to exclude beta fields from the `google` provider.
 
 All resources and fields must be present in meta.yaml files for the provider(s) they're available in.
 
@@ -21,7 +21,7 @@ The name of the Terraform resource. For example, "google_cloudfunctions2_functio
 
 ### `generation_type`
 
-The generation method used to create the Terraform resource. For example, "mmv1", "dcl", "handwritten".
+The generation method used to create the Terraform resource. For example, "mmv1" or "handwritten".
 
 ### `api_service_name`
 

mmv1/third_party/terraform/go.mod

@@ -7,7 +7,6 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.8
 	cloud.google.com/go/bigquery v1.73.1
 	cloud.google.com/go/bigtable v1.42.0
-	github.com/GoogleCloudPlatform/declarative-resource-client-library v1.84.0
 	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/dnaeon/go-vcr v1.0.1

mmv1/third_party/terraform/provider/provider_dcl_resources.go

@@ -0,0 +1,37 @@
+package provider
+
+import (
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	"github.com/hashicorp/terraform-provider-google/google/services/apikeys"
+	"github.com/hashicorp/terraform-provider-google/google/services/assuredworkloads"
+	"github.com/hashicorp/terraform-provider-google/google/services/cloudbuild"
+	"github.com/hashicorp/terraform-provider-google/google/services/clouddeploy"
+	"github.com/hashicorp/terraform-provider-google/google/services/containeraws"
+	"github.com/hashicorp/terraform-provider-google/google/services/containerazure"
+	"github.com/hashicorp/terraform-provider-google/google/services/dataplex"
+	"github.com/hashicorp/terraform-provider-google/google/services/dataproc"
+	"github.com/hashicorp/terraform-provider-google/google/services/firebaserules"
+	"github.com/hashicorp/terraform-provider-google/google/services/gkehub"
+	"github.com/hashicorp/terraform-provider-google/google/services/recaptchaenterprise"
+)
+
+var dclResources = map[string]*schema.Resource{
+	"google_apikeys_key":                   apikeys.ResourceApikeysKey(),
+	"google_assured_workloads_workload":    assuredworkloads.ResourceAssuredWorkloadsWorkload(),
+	"google_cloudbuild_worker_pool":        cloudbuild.ResourceCloudbuildWorkerPool(),
+	"google_clouddeploy_delivery_pipeline": clouddeploy.ResourceClouddeployDeliveryPipeline(),
+	"google_clouddeploy_target":            clouddeploy.ResourceClouddeployTarget(),
+	"google_container_aws_cluster":         containeraws.ResourceContainerAwsCluster(),
+	"google_container_aws_node_pool":       containeraws.ResourceContainerAwsNodePool(),
+	"google_container_azure_client":        containerazure.ResourceContainerAzureClient(),
+	"google_container_azure_cluster":       containerazure.ResourceContainerAzureCluster(),
+	"google_container_azure_node_pool":     containerazure.ResourceContainerAzureNodePool(),
+	"google_dataplex_asset":                dataplex.ResourceDataplexAsset(),
+	"google_dataplex_lake":                 dataplex.ResourceDataplexLake(),
+	"google_dataplex_zone":                 dataplex.ResourceDataplexZone(),
+	"google_dataproc_workflow_template":    dataproc.ResourceDataprocWorkflowTemplate(),
+	"google_firebaserules_release":         firebaserules.ResourceFirebaserulesRelease(),
+	"google_firebaserules_ruleset":         firebaserules.ResourceFirebaserulesRuleset(),
+	"google_gke_hub_feature_membership":    gkehub.ResourceGkeHubFeatureMembership(),
+	"google_recaptcha_enterprise_key":      recaptchaenterprise.ResourceRecaptchaEnterpriseKey(),
+}

mmv1/third_party/terraform/services/apikeys/apikeys_utils.go

@@ -0,0 +1,101 @@
+package apikeys
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"time"
+
+	dcl "github.com/hashicorp/terraform-provider-google/google/tpgdclresource"
+	"google.golang.org/api/googleapi"
+)
+
+func keyStringGetURL(userBasePath string, r *Key) (string, error) {
+	nr := r.urlNormalized()
+	params := map[string]any{
+		"project": dcl.ValueOrEmptyString(nr.Project),
+		"name":    dcl.ValueOrEmptyString(nr.Name),
+	}
+	return dcl.URL("projects/{{project}}/locations/global/keys/{{name}}/keyString", "https://apikeys.googleapis.com/v2/", userBasePath, params), nil
+}
+
+func (c *Client) getKeyStringRaw(ctx context.Context, r *Key) ([]byte, error) {
+
+	u, err := keyStringGetURL(c.Config.BasePath, r)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := dcl.SendRequest(ctx, c.Config, "GET", u, &bytes.Buffer{}, c.Config.RetryProvider)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Response.Body.Close()
+	b, err := io.ReadAll(resp.Response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func (c *Client) getKeyRaw(ctx context.Context, r *Key) ([]byte, error) {
+
+	u, err := r.getURL(c.Config.BasePath)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := dcl.SendRequest(ctx, c.Config, "GET", u, &bytes.Buffer{}, c.Config.RetryProvider)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Response.Body.Close()
+	b, err := io.ReadAll(resp.Response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func (c *Client) GetKey(ctx context.Context, r *Key) (*Key, error) {
+	ctx, cancel := context.WithTimeout(ctx, c.Config.TimeoutOr(60*time.Second))
+	defer cancel()
+
+	b, err := c.getKeyRaw(ctx, r)
+	if err != nil {
+		if dcl.IsNotFound(err) {
+			return nil, &googleapi.Error{
+				Code:    404,
+				Message: err.Error(),
+			}
+		}
+		return nil, err
+	}
+	result, err := unmarshalKey(b, c, r)
+	if err != nil {
+		return nil, err
+	}
+	// Get the value of KeyString through a separate api method.
+	b, err = c.getKeyStringRaw(ctx, r)
+	if err != nil {
+		return nil, err
+	}
+	var m map[string]any
+	if err := json.Unmarshal(b, &m); err != nil {
+		return nil, err
+	}
+	result.KeyString = dcl.FlattenString(m["keyString"])
+	result.Project = r.Project
+	result.Name = r.Name
+
+	c.Config.Logger.Infof("Retrieved raw result state: %v", result)
+	c.Config.Logger.Infof("Canonicalizing with specified state: %v", r)
+	result, err = canonicalizeKeyNewState(c, result, r)
+	if err != nil {
+		return nil, err
+	}
+	c.Config.Logger.Infof("Created result state: %v", result)
+
+	return result, nil
+}

mmv1/third_party/terraform/services/apikeys/client.go

@@ -0,0 +1,18 @@
+package apikeys
+
+import (
+	dcl "github.com/hashicorp/terraform-provider-google/google/tpgdclresource"
+)
+
+// The Client is the base struct of all operations.  This will receive the
+// Get, Delete, List, and Apply operations on all resources.
+type Client struct {
+	Config *dcl.Config
+}
+
+// NewClient creates a client that retries all operations a few times each.
+func NewClient(c *dcl.Config) *Client {
+	return &Client{
+		Config: c,
+	}
+}