Skip to content

Commit

Permalink
Enhancements to YBD payload after assessment refactoring (#2238)
Browse files Browse the repository at this point in the history 
* Bumped up the payload version to 1.2
* Updated the AssessmentIssue payload for YBD to be in sync with standard struct format
* Adding MigrationComplexityExplanation to ybd assessment payload
  • Loading branch information
@sanyamsinghal
sanyamsinghal authored Jan 23, 2025
1 parent 7a68e5e commit 6bf1539
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 102 deletions.
124 changes: 36 additions & 88 deletions yb-voyager/cmd/assessMigrationCommand.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) {
// whatever happens later will be stored in the struct's field. so to be on safer side, we will build it again here as per required format.
explanation, err := buildMigrationComplexityExplanation(source.DBType, assessmentReport, "")
if err != nil {
log.Errorf("failed to build migration complexity explanation: %v", err)
log.Errorf("failed to build migration complexity explanation for callhome assessment payload: %v", err)
}

var obfuscatedIssues []callhome.AssessmentIssueCallhome
Expand Down Expand Up @@ -462,15 +462,22 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE
utils.PrintAndLog("failed to calculate the total sharded table size from tableIndexStats: %v", err)
}

assessmentIssues := flattenAssessmentReportToAssessmentIssues(assessmentReport)
assessmentIssues := convertAssessmentIssueToYugabyteDAssessmentIssue(assessmentReport)
// we will build this twice for json and html reports both, at the time of report generation.
// whatever happens later will be stored in the struct's field. so to be on safer side, we will build it again here as per required format.
explanation, err := buildMigrationComplexityExplanation(source.DBType, assessmentReport, "")
if err != nil {
log.Errorf("failed to build migration complexity explanation for yugabyted assessment payload: %v", err)
}

payload := AssessMigrationPayload{
PayloadVersion: ASSESS_MIGRATION_YBD_PAYLOAD_VERSION,
VoyagerVersion: assessmentReport.VoyagerVersion,
TargetDBVersion: assessmentReport.TargetDBVersion,
MigrationComplexity: assessmentReport.MigrationComplexity,
SchemaSummary: assessmentReport.SchemaSummary,
AssessmentIssues: assessmentIssues,
PayloadVersion: ASSESS_MIGRATION_YBD_PAYLOAD_VERSION,
VoyagerVersion: assessmentReport.VoyagerVersion,
TargetDBVersion: assessmentReport.TargetDBVersion,
MigrationComplexity: assessmentReport.MigrationComplexity,
MigrationComplexityExplanation: explanation,
SchemaSummary: assessmentReport.SchemaSummary,
AssessmentIssues: assessmentIssues,
SourceSizeDetails: SourceDBSizeDetails{
TotalIndexSize: assessmentReport.GetTotalIndexSize(),
TotalTableSize: assessmentReport.GetTotalTableSize(),
Expand Down Expand Up @@ -510,84 +517,25 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE
return ev
}

// flatten UnsupportedDataTypes, UnsupportedFeatures, MigrationCaveats
func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []AssessmentIssueYugabyteD {
var issues []AssessmentIssueYugabyteD

var dataTypesDocsLink string
switch source.DBType {
case POSTGRESQL:
dataTypesDocsLink = UNSUPPORTED_DATATYPES_DOC_LINK
case ORACLE:
dataTypesDocsLink = UNSUPPORTED_DATATYPES_DOC_LINK_ORACLE
}
for _, unsupportedDataType := range ar.UnsupportedDataTypes {
issues = append(issues, AssessmentIssueYugabyteD{
Type: constants.DATATYPE,
TypeDescription: GetCategoryDescription(constants.DATATYPE),
Subtype: unsupportedDataType.DataType,
ObjectName: fmt.Sprintf("%s.%s.%s", unsupportedDataType.SchemaName, unsupportedDataType.TableName, unsupportedDataType.ColumnName),
SqlStatement: "",
DocsLink: dataTypesDocsLink,
})
}

for _, unsupportedFeature := range ar.UnsupportedFeatures {
for _, object := range unsupportedFeature.Objects {
issues = append(issues, AssessmentIssueYugabyteD{
Type: constants.FEATURE,
TypeDescription: GetCategoryDescription(constants.FEATURE),
Subtype: unsupportedFeature.FeatureName,
SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: unsupportedFeature.DocsLink,
MinimumVersionsFixedIn: unsupportedFeature.MinimumVersionsFixedIn,
})
}
}

for _, migrationCaveat := range ar.MigrationCaveats {
for _, object := range migrationCaveat.Objects {
issues = append(issues, AssessmentIssueYugabyteD{
Type: constants.MIGRATION_CAVEATS,
TypeDescription: GetCategoryDescription(constants.MIGRATION_CAVEATS),
Subtype: migrationCaveat.FeatureName,
SubtypeDescription: migrationCaveat.FeatureDescription,
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: migrationCaveat.DocsLink,
MinimumVersionsFixedIn: migrationCaveat.MinimumVersionsFixedIn,
})
}
}

for _, uqc := range ar.UnsupportedQueryConstructs {
issues = append(issues, AssessmentIssueYugabyteD{
Type: constants.QUERY_CONSTRUCT,
TypeDescription: GetCategoryDescription(constants.QUERY_CONSTRUCT),
Subtype: uqc.ConstructTypeName,
SqlStatement: uqc.Query,
DocsLink: uqc.DocsLink,
MinimumVersionsFixedIn: uqc.MinimumVersionsFixedIn,
})
}

for _, plpgsqlObjects := range ar.UnsupportedPlPgSqlObjects {
for _, object := range plpgsqlObjects.Objects {
issues = append(issues, AssessmentIssueYugabyteD{
Type: constants.PLPGSQL_OBJECT,
TypeDescription: GetCategoryDescription(constants.PLPGSQL_OBJECT),
Subtype: plpgsqlObjects.FeatureName,
SubtypeDescription: plpgsqlObjects.FeatureDescription,
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: plpgsqlObjects.DocsLink,
MinimumVersionsFixedIn: plpgsqlObjects.MinimumVersionsFixedIn,
})
func convertAssessmentIssueToYugabyteDAssessmentIssue(ar AssessmentReport) []AssessmentIssueYugabyteD {
var result []AssessmentIssueYugabyteD
for _, issue := range ar.Issues {
ybdIssue := AssessmentIssueYugabyteD{
Category: issue.Category,
CategoryDescription: issue.CategoryDescription,
Type: issue.Type, // Ques: should we be just sending Name in AssessmentIssueYugabyteD payload
Name: issue.Name,
Description: issue.Description,
Impact: issue.Impact,
ObjectType: issue.ObjectType,
ObjectName: issue.ObjectName,
SqlStatement: issue.SqlStatement,
DocsLink: issue.DocsLink,
MinimumVersionsFixedIn: issue.MinimumVersionsFixedIn,
}
result = append(result, ybdIssue)
}
return issues
return result
}

func runAssessment() error {
Expand Down Expand Up @@ -1229,10 +1177,10 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U
})
}
feature := UnsupportedFeature{
FeatureName: issueName,
DisplayDDL: true,
DocsLink: docsLink,
Objects: objects,
FeatureName: issueName,
DisplayDDL: true,
DocsLink: docsLink,
Objects: objects,
MinimumVersionsFixedIn: minVersionsFixedIn,
}
unsupportedPlpgSqlObjects = append(unsupportedPlpgSqlObjects, feature)
Expand Down
38 changes: 24 additions & 14 deletions yb-voyager/cmd/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1129,27 +1129,37 @@ type AssessMigrationDBConfig struct {
// =============== for yugabyted controlplane ==============//
// TODO: see if this can be accommodated in controlplane pkg, facing pkg cyclic dependency issue

var ASSESS_MIGRATION_YBD_PAYLOAD_VERSION = "1.1" // version(s) till now: 1.0, 1.1
/*
Version History
1.0: Introduced AssessmentIssue field for storing assessment issues in flattened format
1.1: Added TargetDBVersion and AssessmentIssueYugabyteD.MinimumVersionFixedIn
1.2: Syncing it with original AssessmentIssue(adding fields Category, CategoryDescription, Type, Name, Description, Impact, ObjectType) and MigrationComplexityExplanation;
*/
var ASSESS_MIGRATION_YBD_PAYLOAD_VERSION = "1.2"

type AssessMigrationPayload struct {
PayloadVersion string
VoyagerVersion string
TargetDBVersion *ybversion.YBVersion
MigrationComplexity string
SchemaSummary utils.SchemaSummary
AssessmentIssues []AssessmentIssueYugabyteD
SourceSizeDetails SourceDBSizeDetails
TargetRecommendations TargetSizingRecommendations
ConversionIssues []utils.AnalyzeSchemaIssue
PayloadVersion string
VoyagerVersion string
TargetDBVersion *ybversion.YBVersion
MigrationComplexity string
MigrationComplexityExplanation string
SchemaSummary utils.SchemaSummary
AssessmentIssues []AssessmentIssueYugabyteD
SourceSizeDetails SourceDBSizeDetails
TargetRecommendations TargetSizingRecommendations
ConversionIssues []utils.AnalyzeSchemaIssue
// Depreacted: AssessmentJsonReport is depricated; use the fields directly inside struct
AssessmentJsonReport AssessmentReportYugabyteD
}

type AssessmentIssueYugabyteD struct {
Type string `json:"Type"` // Feature, DataType, MigrationCaveat, UQC
TypeDescription string `json:"TypeDescription"` // Based on AssessmentIssue type
Subtype string `json:"Subtype"` // GIN Indexes, Advisory Locks etc
SubtypeDescription string `json:"SubtypeDescription"` // description based on subtype
Category string `json:"Category"` // expected values: unsupported_features, unsupported_query_constructs, migration_caveats, unsupported_plpgsql_objects, unsupported_datatype
CategoryDescription string `json:"CategoryDescription"`
Type string `json:"Type"` // Ex: GIN_INDEXES, SECURITY_INVOKER_VIEWS, STORED_GENERATED_COLUMNS
Name string `json:"Name"` // Ex: GIN Indexes, Security Invoker Views, Stored Generated Columns
Description string `json:"Description"` // description based on type/name
Impact string `json:"Impact"` // // Level-1, Level-2, Level-3 (no default: need to be assigned for each issue)
ObjectType string `json:"ObjectType"` // For datatype category, ObjectType will be datatype (for eg "geometry")
ObjectName string `json:"ObjectName"` // Fully qualified object name(empty if NA, eg UQC)
SqlStatement string `json:"SqlStatement"` // DDL or DML(UQC)
DocsLink string `json:"DocsLink"` // docs link based on the subtype
Expand Down

0 comments on commit 6bf1539

Please sign in to comment.