tools/importer-rest-api-specs: refactoring the normalization pack…

…age into the new structure

tools/importer-rest-api-specs/components/parser/load_and_parse.go

@@ -7,8 +7,10 @@ import (
 	"fmt"
 	"strings"
 
+	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/dataworkarounds"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/resourceids"
+	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/components/apidefinitions/parser/cleanup"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/logging"
 	importerModels "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models"
 )
@@ -92,7 +94,16 @@ func LoadAndParseFiles(directory string, fileNames []string, serviceName, apiVer
 	out = *output
 
 	for _, service := range out {
-		service.Resources = removeUnusedItems(service.Resources)
+		// temporary shim to enable using the new logic
+		version := sdkModels.APIVersion{
+			APIVersion: service.ApiVersion,
+			Generate:   true,
+			Preview:    service.IsPreviewVersion(),
+			Resources:  service.Resources,
+			Source:     sdkModels.AzureRestAPISpecsSourceDataOrigin,
+		}
+		version = cleanup.RemoveUnusedItems(version)
+		service.Resources = version.Resources
 	}
 
 	if len(out) > 1 {

tools/importer-rest-api-specs/components/parser/operations.go

@@ -11,9 +11,10 @@ import (
 
 	"github.com/go-openapi/spec"
 	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
-	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/cleanup"
+	legacyCleanup "github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/cleanup"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/internal"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/resourceids"
+	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/components/apidefinitions/parser/cleanup"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/components/apidefinitions/parser/constants"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/logging"
 )
@@ -350,7 +351,7 @@ func (p operationsParser) optionsForOperation(input parsedOperation) (*map[strin
 
 		if strings.EqualFold(param.In, "header") || strings.EqualFold(param.In, "query") {
 			val := param.Name
-			name := cleanup.NormalizeName(val)
+			name := legacyCleanup.NormalizeName(val)
 
 			option := sdkModels.SDKOperationOption{
 				Required: param.Required,
@@ -537,7 +538,7 @@ func (d *SwaggerDefinition) findOperationsMatchingTag(tag *string) *[]parsedOper
 				continue
 			}
 
-			operationName := normalizeOperationName(operationDetails.ID, tag)
+			operationName := cleanup.NormalizeOperationName(operationDetails.ID, tag)
 			result = append(result, parsedOperation{
 				name:       operationName,
 				uri:        uri,

tools/importer-rest-api-specs/components/parser/parser.go

@@ -10,8 +10,9 @@ import (
 	"github.com/hashicorp/go-azure-helpers/lang/pointer"
 	"github.com/hashicorp/pandora/tools/data-api-sdk/v1/helpers"
 	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
-	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/cleanup"
+	legacyCleanup "github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/cleanup"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/resourceids"
+	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/components/apidefinitions/parser/cleanup"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/logging"
 	importerModels "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models"
 )
@@ -33,8 +34,8 @@ func (d *SwaggerDefinition) parse(serviceName, apiVersion string, resourceProvid
 
 		if resource != nil {
 			logging.Tracef("The Tag %q has %d API Operations", tag, len(resource.Operations))
-			normalizedTag := normalizeTag(tag)
-			normalizedTag = cleanup.NormalizeResourceName(normalizedTag)
+			normalizedTag := cleanup.NormalizeTag(tag)
+			normalizedTag = legacyCleanup.NormalizeResourceName(normalizedTag)
 			resources[normalizedTag] = *resource
 		}
 	}
@@ -48,11 +49,11 @@ func (d *SwaggerDefinition) parse(serviceName, apiVersion string, resourceProvid
 
 		// Since we're dealing with missing tag data in the swagger, we'll assume the proper tag name here is the file name
 		// This is less than ideal, but _should_ be fine.
-		inferredTag := cleanup.PluraliseName(d.Name)
+		inferredTag := legacyCleanup.PluraliseName(d.Name)
 
 		if resource != nil {
-			normalizedTag := normalizeTag(inferredTag)
-			normalizedTag = cleanup.NormalizeResourceName(normalizedTag)
+			normalizedTag := cleanup.NormalizeTag(inferredTag)
+			normalizedTag = legacyCleanup.NormalizeResourceName(normalizedTag)
 
 			if mergeResources, ok := resources[normalizedTag]; ok {
 				resources[normalizedTag] = importerModels.MergeResourcesForTag(mergeResources, *resource)
@@ -76,8 +77,8 @@ func (d *SwaggerDefinition) parse(serviceName, apiVersion string, resourceProvid
 	swaggerFileName := strings.Split(d.Name, "/")
 	if len(resources) == 0 && len(swaggerFileName) > 2 {
 		// if we're here then there is no tag in this file, so we'll use the file name
-		inferredTag := cleanup.PluraliseName(swaggerFileName[len(swaggerFileName)-1])
-		normalizedTag := cleanup.NormalizeResourceName(inferredTag)
+		inferredTag := legacyCleanup.PluraliseName(swaggerFileName[len(swaggerFileName)-1])
+		normalizedTag := legacyCleanup.NormalizeResourceName(inferredTag)
 
 		result, err := d.findOrphanedDiscriminatedModels(serviceName)
 		if err != nil {
@@ -90,7 +91,7 @@ func (d *SwaggerDefinition) parse(serviceName, apiVersion string, resourceProvid
 				Constants: result.Constants,
 				Models:    result.Models,
 			}
-			resource = normalizeAzureApiResource(resource)
+			resource = cleanup.NormalizeAPIResource(resource)
 
 			if mergeResources, ok := resources[normalizedTag]; ok {
 				resources[normalizedTag] = importerModels.MergeResourcesForTag(mergeResources, resource)
@@ -101,7 +102,7 @@ func (d *SwaggerDefinition) parse(serviceName, apiVersion string, resourceProvid
 	}
 
 	return &importerModels.AzureApiDefinition{
-		ServiceName: cleanup.NormalizeServiceName(serviceName),
+		ServiceName: legacyCleanup.NormalizeServiceName(serviceName),
 		ApiVersion:  apiVersion,
 		Resources:   resourcesOut,
 	}, nil

tools/importer-rest-api-specs/components/parser/swagger_resources.go

@@ -14,6 +14,7 @@ import (
 	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/internal"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/resourceids"
+	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/components/apidefinitions/parser/cleanup"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/components/apidefinitions/parser/commonschema"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/components/apidefinitions/parser/constants"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/internal/logging"
@@ -70,7 +71,7 @@ func (d *SwaggerDefinition) parseResourcesWithinSwaggerTag(tag *string, resource
 	resource = switchOutCustomTypesAsNeeded(resource)
 
 	// first Normalize the names, meaning `foo` -> `Foo` for consistency
-	resource = normalizeAzureApiResource(resource)
+	resource = cleanup.NormalizeAPIResource(resource)
 
 	return &resource, nil
 }

tools/importer-rest-api-specs/components/parser/normalizer.go → tools/importer-rest-api-specs/internal/components/apidefinitions/parser/cleanup/normalization.go

@@ -1,18 +1,21 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: MPL-2.0
 
-package parser
+package cleanup
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/hashicorp/go-azure-helpers/lang/pointer"
 	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
 	"github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/cleanup"
 )
 
-// normalizeAzureApiResource works through the parsed AzureApiResource and ensures
+// NormalizeAPIResource works through the parsed AzureApiResource and ensures
 // that all the Names and References are consistent (TitleCase) as a final effort
 // to ensure the Swagger Data is normalized.
-func normalizeAzureApiResource(input sdkModels.APIResource) sdkModels.APIResource {
+func NormalizeAPIResource(input sdkModels.APIResource) sdkModels.APIResource {
 	normalizedConstants := make(map[string]sdkModels.SDKConstant)
 	for k, v := range input.Constants {
 		name := cleanup.NormalizeName(k)
@@ -65,7 +68,7 @@ func normalizeAzureApiResource(input sdkModels.APIResource) sdkModels.APIResourc
 		for optionKey, optionVal := range v.Options {
 			optionKey = cleanup.NormalizeName(optionKey)
 
-			optionVal.ObjectDefinition = normalizeOptionsObjectDefinition(optionVal.ObjectDefinition)
+			optionVal.ObjectDefinition = normalizeSDKOptionsObjectDefinition(optionVal.ObjectDefinition)
 
 			normalizedOptions[optionKey] = optionVal
 		}
@@ -119,16 +122,54 @@ func normalizeSDKObjectDefinition(input sdkModels.SDKObjectDefinition) sdkModels
 	return input
 }
 
-func normalizeOptionsObjectDefinition(input sdkModels.SDKOperationOptionObjectDefinition) sdkModels.SDKOperationOptionObjectDefinition {
+func normalizeSDKOptionsObjectDefinition(input sdkModels.SDKOperationOptionObjectDefinition) sdkModels.SDKOperationOptionObjectDefinition {
 	if input.ReferenceName != nil {
 		normalized := cleanup.NormalizeName(*input.ReferenceName)
 		input.ReferenceName = &normalized
 	}
 
 	if input.NestedItem != nil {
-		nestedItem := normalizeOptionsObjectDefinition(*input.NestedItem)
+		nestedItem := normalizeSDKOptionsObjectDefinition(*input.NestedItem)
 		input.NestedItem = pointer.To(nestedItem)
 	}
 
 	return input
 }
+
+func NormalizeOperationName(operationId string, tag *string) string {
+	operationName := operationId
+	// in some cases the OperationId *is* the Tag, in this instance I guess we take that as ok?
+	if tag != nil && !strings.EqualFold(operationName, *tag) {
+		// we're intentionally not using `strings.TrimPrefix` here since we want
+		// to account for the casing of the tag being different
+		if strings.HasPrefix(strings.ToLower(operationName), strings.ToLower(*tag)) {
+			operationName = operationName[len(*tag):]
+
+			// however if the Tag is `ManagementGroupsSubscriptions`, then we need to keep the extra `S` around
+			if *tag == "ManagementGroups" && !strings.HasPrefix(operationName, "_") {
+				operationName = fmt.Sprintf("S%s", operationName)
+			}
+		}
+	}
+	operationName = strings.ReplaceAll(operationName, "_", "")
+	operationName = strings.TrimPrefix(operationName, "Operations") // sanity checking
+	if !strings.HasPrefix(strings.ToLower(operationName), "subscriptions") {
+		operationName = strings.TrimPrefix(operationName, "s") // plurals
+	}
+	operationName = strings.TrimPrefix(operationName, "_")
+	operationName = cleanup.NormalizeName(operationName)
+	return operationName
+}
+
+func NormalizeTag(input string) string {
+	// NOTE: we could be smarter here, but given this is a handful of cases it's
+	// probably prudent to hard-code these for now (and fix the swaggers as we
+	// come across them?)
+	output := input
+	output = strings.ReplaceAll(output, "EndPoint", "Endpoint")
+	output = strings.ReplaceAll(output, "NetWork", "Network")
+	output = strings.ReplaceAll(output, "Baremetalinfrastructure", "BareMetalInfrastructure")
+	output = strings.ReplaceAll(output, "VirtualWans", "VirtualWANs")
+
+	return output
+}

tools/importer-rest-api-specs/internal/components/apidefinitions/parser/cleanup/remove_unused_items.go

@@ -1,3 +1,6 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
 package cleanup
 
 import (
@@ -7,7 +10,7 @@ import (
 	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
 )
 
-func RemoveUnusedItems(input sdkModels.APIVersion) (*sdkModels.APIVersion, error) {
+func RemoveUnusedItems(input sdkModels.APIVersion) sdkModels.APIVersion {
 	// The ordering matters here, we need to remove the ResourceIDs first since
 	// they contain references to Constants - as do Models, so remove unused
 	// Resource IDs, then Models, then Constants else we can have orphaned
@@ -59,7 +62,7 @@ func RemoveUnusedItems(input sdkModels.APIVersion) (*sdkModels.APIVersion, error
 	}
 
 	input.Resources = outputResources
-	return &input, nil
+	return input
 }
 
 func findUnusedConstants(operations map[string]sdkModels.SDKOperation, resourceIds map[string]sdkModels.ResourceID, resourceModels map[string]sdkModels.SDKModel, resourceConstants map[string]sdkModels.SDKConstant) []string {

tools/importer-rest-api-specs/internal/components/apidefinitions/parser/parser.go

@@ -33,19 +33,16 @@ func ParseAPIVersion(serviceName string, input discoveryModels.AvailableDataSetF
 
 	// Next let's apply any data workarounds
 	logging.Debugf("Applying Data Workarounds..")
-	output, err := dataworkarounds.Apply(serviceName, apiVersion)
+	withFixesApplied, err := dataworkarounds.Apply(serviceName, apiVersion)
 	if err != nil {
 		return nil, fmt.Errorf("applying Data Workarounds for Service %q / API Version %q: %+v", serviceName, input.APIVersion, err)
 	}
 	logging.Debugf("Applying Data Workarounds - Complete.")
 
 	// Finally let's remove any unused items
 	logging.Debugf("Removing unused items..")
-	output, err = cleanup.RemoveUnusedItems(*output)
-	if err != nil {
-		return nil, fmt.Errorf("removing unused items from Service %q / API Version %q: %+v", serviceName, input.APIVersion, err)
-	}
+	output := cleanup.RemoveUnusedItems(*withFixesApplied)
 	logging.Debugf("Removing unused items - Complete.")
 
-	return output, nil
+	return &output, nil
 }