Skip to content

Commit

Permalink
Implement versioning for Assess Migration callhome payload (#2234)
Browse files Browse the repository at this point in the history 
- it starts with 1.0
- with this version we are also sending MigrationComplexityExplanation and assessment issues struct instead of separate fields for each category to callhome

* Updated Assessment callhome struct definition tests for upgrade
* Fixed a typo in Assessment Issue field name to be consistent
  • Loading branch information
@sanyamsinghal
sanyamsinghal authored Jan 23, 2025
1 parent 37fb28b commit ae842f5
Show file tree
Hide file tree
Showing 10 changed files with 215 additions and 213 deletions.
8 changes: 4 additions & 4 deletions migtests/tests/analyze-schema/expected_issues.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@
"ObjectName": "test_1, constraint: (test_1_id_fkey)",
"Reason": "Foreign key references to partitioned table are not yet supported in YugabyteDB.",
"SqlStatement": "CREATE TABLE test_1 (\n\tid numeric NOT NULL REFERENCES sales_data(sales_id),\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id)\n) PARTITION BY LIST (country_code, record_type) ;",
"Suggestion": "No workaround available ",
"Suggestion": "No workaround available.",
"GH": "https://github.com/yugabyte/yugabyte-db/issues/25575",
"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features",
"MinimumVersionsFixedIn": {
Expand Down Expand Up @@ -2093,7 +2093,7 @@
"ObjectName": "public.asterisks1",
"Reason": "SQL Body for sql languages in function statement is not supported in YugabyteDB",
"SqlStatement": "CREATE FUNCTION public.asterisks1(n integer) RETURNS text\n LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE\n RETURN repeat('*'::text, n);",
"Suggestion": "No workaround available",
"Suggestion": "No workaround available.",
"GH": "https://github.com/yugabyte/yugabyte-db/issues/25575",
"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features",
"MinimumVersionsFixedIn": {
Expand All @@ -2106,7 +2106,7 @@
"ObjectName": "public.asterisks",
"Reason": "SQL Body for sql languages in function statement is not supported in YugabyteDB",
"SqlStatement": "CREATE FUNCTION public.asterisks(n integer) RETURNS SETOF text\n LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE\n BEGIN ATOMIC\n SELECT repeat('*'::text, g.g) AS repeat\n FROM generate_series(1, asterisks.n) g(g);\nEND;",
"Suggestion": "No workaround available",
"Suggestion": "No workaround available.",
"GH": "https://github.com/yugabyte/yugabyte-db/issues/25575",
"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features",
"MinimumVersionsFixedIn": {
Expand All @@ -2119,7 +2119,7 @@
"ObjectName": "add",
"Reason": "SQL Body for sql languages in function statement is not supported in YugabyteDB",
"SqlStatement": "CREATE FUNCTION add(int, int) RETURNS int IMMUTABLE PARALLEL SAFE BEGIN ATOMIC; SELECT $1 + $2; END;",
"Suggestion": "No workaround available",
"Suggestion": "No workaround available.",
"GH": "https://github.com/yugabyte/yugabyte-db/issues/25575",
"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features",
"MinimumVersionsFixedIn": {
Expand Down
1 change: 0 additions & 1 deletion yb-voyager/cmd/analyzeSchema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1274,7 +1274,6 @@ func packAndSendAnalyzeSchemaPayload(status string, errorMsg string) {
if sensitiveReason == UNSUPPORTED_PG_SYNTAX_ISSUE_REASON && strings.HasPrefix(issue.Reason, sensitiveReason) {
issue.Reason = sensitiveReason
} else {
// TODO: should we just start sending issue type/name here instead of obfuscating the reason since that is anyways static
match, err := utils.MatchesFormatString(sensitiveReason, issue.Reason)
if match {
issue.Reason, err = utils.ObfuscateFormatDetails(sensitiveReason, issue.Reason, constants.OBFUSCATE_STRING)
Expand Down
196 changes: 83 additions & 113 deletions yb-voyager/cmd/assessMigrationCommand.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,23 +120,29 @@ var assessMigrationCmd = &cobra.Command{
},
}

// Assessment feature names to send the object names for to callhome
var featuresToSendObjectsToCallhome = []string{}

func packAndSendAssessMigrationPayload(status string, errMsg string) {
if !shouldSendCallhome() {
return
}
payload := createCallhomePayload()

payload := createCallhomePayload()
payload.MigrationPhase = ASSESS_MIGRATION_PHASE
payload.Status = status
if assessmentMetadataDirFlag == "" {
sourceDBDetails := callhome.SourceDBDetails{
DBType: source.DBType,
DBVersion: source.DBVersion,
DBSize: source.DBSize,
}
payload.SourceDBDetails = callhome.MarshalledJsonString(sourceDBDetails)
}

var tableSizingStats, indexSizingStats []callhome.ObjectSizingStats
if assessmentReport.TableIndexStats != nil {
for _, stat := range *assessmentReport.TableIndexStats {
newStat := callhome.ObjectSizingStats{
//redacting schema and object name
ObjectName: "XXX",
ObjectName: constants.OBFUSCATE_STRING,
ReadsPerSecond: utils.SafeDereferenceInt64(stat.ReadsPerSecond),
WritesPerSecond: utils.SafeDereferenceInt64(stat.WritesPerSecond),
SizeInBytes: utils.SafeDereferenceInt64(stat.SizeInBytes),
Expand All @@ -156,87 +162,51 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) {
}),
}

unsupportedDatatypesList := lo.Map(assessmentReport.UnsupportedDataTypes, func(datatype utils.TableColumnsDataTypes, _ int) string {
return datatype.DataType
})
// we will build this twice for json and html reports both, at the time of report generation.
// whatever happens later will be stored in the struct's field. so to be on safer side, we will build it again here as per required format.
explanation, err := buildMigrationComplexityExplanation(source.DBType, assessmentReport, "")
if err != nil {
log.Errorf("failed to build migration complexity explanation: %v", err)
}

var obfuscatedIssues []callhome.AssessmentIssueCallhome
for _, issue := range assessmentReport.Issues {
obfuscatedIssue := callhome.AssessmentIssueCallhome{
Category: issue.Category,
CategoryDescription: issue.CategoryDescription,
Type: issue.Type,
Name: issue.Name,
Impact: issue.Impact,
ObjectType: issue.ObjectType,
}

groupedByConstructType := lo.GroupBy(assessmentReport.UnsupportedQueryConstructs, func(q utils.UnsupportedQueryConstruct) string {
return q.ConstructTypeName
})
countByConstructType := lo.MapValues(groupedByConstructType, func(constructs []utils.UnsupportedQueryConstruct, _ string) int {
return len(constructs)
})
// TODO: object name(for extensions) and other details like datatype/indextype(present in description) for callhome will be covered in next release

var unsupportedFeatures []callhome.UnsupportedFeature
for _, feature := range assessmentReport.UnsupportedFeatures {
if feature.FeatureName == EXTENSION_FEATURE {
// For extensions, we need to send the extension name in the feature name
// for better categorization in callhome
for _, object := range feature.Objects {
unsupportedFeatures = append(unsupportedFeatures, callhome.UnsupportedFeature{
FeatureName: fmt.Sprintf("%s - %s", feature.FeatureName, object.ObjectName),
ObjectCount: 1,
TotalOccurrences: 1,
})
}
} else {
var objects []string
if slices.Contains(featuresToSendObjectsToCallhome, feature.FeatureName) {
objects = lo.Map(feature.Objects, func(o ObjectInfo, _ int) string {
return o.ObjectName
})
}
unsupportedFeatures = append(unsupportedFeatures, callhome.UnsupportedFeature{
FeatureName: feature.FeatureName,
ObjectCount: len(feature.Objects),
Objects: objects,
TotalOccurrences: len(feature.Objects),
})
}
// if issue.Type == UNSUPPORTED_EXTENSION_ISSUE_TYPE {
// object name(i.e. extension name here) might be qualified with schema so just taking the last part
// obfuscatedIssue.ObjectName = strings.Split(issue.ObjectName, ".")[len(strings.Split(issue.ObjectName, "."))-1]
// }

// appending the issue after obfuscating sensitive information
obfuscatedIssues = append(obfuscatedIssues, obfuscatedIssue)
}

assessPayload := callhome.AssessMigrationPhasePayload{
TargetDBVersion: assessmentReport.TargetDBVersion,
MigrationComplexity: assessmentReport.MigrationComplexity,
UnsupportedFeatures: callhome.MarshalledJsonString(unsupportedFeatures),
UnsupportedQueryConstructs: callhome.MarshalledJsonString(countByConstructType),
UnsupportedDatatypes: callhome.MarshalledJsonString(unsupportedDatatypesList),
MigrationCaveats: callhome.MarshalledJsonString(lo.Map(assessmentReport.MigrationCaveats, func(feature UnsupportedFeature, _ int) callhome.UnsupportedFeature {
return callhome.UnsupportedFeature{
FeatureName: feature.FeatureName,
ObjectCount: len(feature.Objects),
TotalOccurrences: len(feature.Objects),
}
})),
UnsupportedPlPgSqlObjects: callhome.MarshalledJsonString(lo.Map(assessmentReport.UnsupportedPlPgSqlObjects, func(plpgsql UnsupportedFeature, _ int) callhome.UnsupportedFeature {
groupedObjects := groupByObjectName(plpgsql.Objects)
return callhome.UnsupportedFeature{
FeatureName: plpgsql.FeatureName,
ObjectCount: len(lo.Keys(groupedObjects)),
TotalOccurrences: len(plpgsql.Objects),
}
})),
TableSizingStats: callhome.MarshalledJsonString(tableSizingStats),
IndexSizingStats: callhome.MarshalledJsonString(indexSizingStats),
SchemaSummary: callhome.MarshalledJsonString(schemaSummaryCopy),
IopsInterval: intervalForCapturingIOPS,
Error: callhome.SanitizeErrorMsg(errMsg),
PayloadVersion: callhome.ASSESS_MIGRATION_CALLHOME_PAYLOAD_VERSION,
TargetDBVersion: assessmentReport.TargetDBVersion,
MigrationComplexity: assessmentReport.MigrationComplexity,
MigrationComplexityExplanation: explanation,
SchemaSummary: callhome.MarshalledJsonString(schemaSummaryCopy),
Issues: callhome.MarshalledJsonString(obfuscatedIssues),
Error: callhome.SanitizeErrorMsg(errMsg),
TableSizingStats: callhome.MarshalledJsonString(tableSizingStats),
IndexSizingStats: callhome.MarshalledJsonString(indexSizingStats),
SourceConnectivity: assessmentMetadataDirFlag == "",
IopsInterval: intervalForCapturingIOPS,
}

if assessmentMetadataDirFlag == "" {
sourceDBDetails := callhome.SourceDBDetails{
DBType: source.DBType,
DBVersion: source.DBVersion,
DBSize: source.DBSize,
}
payload.SourceDBDetails = callhome.MarshalledJsonString(sourceDBDetails)
assessPayload.SourceConnectivity = true
} else {
assessPayload.SourceConnectivity = false
}
payload.PhasePayload = callhome.MarshalledJsonString(assessPayload)
payload.Status = status
err := callhome.SendPayload(&payload)
err = callhome.SendPayload(&payload)
if err == nil && (status == COMPLETE || status == ERROR) {
callHomeErrorOrCompletePayloadSent = true
}
Expand Down Expand Up @@ -495,7 +465,7 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE
assessmentIssues := flattenAssessmentReportToAssessmentIssues(assessmentReport)

payload := AssessMigrationPayload{
PayloadVersion: ASSESS_MIGRATION_PAYLOAD_VERSION,
PayloadVersion: ASSESS_MIGRATION_YBD_PAYLOAD_VERSION,
VoyagerVersion: assessmentReport.VoyagerVersion,
TargetDBVersion: assessmentReport.TargetDBVersion,
MigrationComplexity: assessmentReport.MigrationComplexity,
Expand Down Expand Up @@ -1038,12 +1008,12 @@ func convertAnalyzeSchemaIssueToAssessmentIssue(analyzeSchemaIssue utils.Analyze
// and we don't use any Suggestion field in AssessmentIssue. Combination of Description + DocsLink should be enough
Description: analyzeSchemaIssue.Reason + " " + analyzeSchemaIssue.Suggestion,

Impact: analyzeSchemaIssue.Impact,
ObjectType: analyzeSchemaIssue.ObjectType,
ObjectName: analyzeSchemaIssue.ObjectName,
SqlStatement: analyzeSchemaIssue.SqlStatement,
DocsLink: analyzeSchemaIssue.DocsLink,
MinimumVersionFixedIn: minVersionsFixedIn,
Impact: analyzeSchemaIssue.Impact,
ObjectType: analyzeSchemaIssue.ObjectType,
ObjectName: analyzeSchemaIssue.ObjectName,
SqlStatement: analyzeSchemaIssue.SqlStatement,
DocsLink: analyzeSchemaIssue.DocsLink,
MinimumVersionsFixedIn: minVersionsFixedIn,
}
}

Expand Down Expand Up @@ -1246,16 +1216,16 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U
docsLink = issue.DocsLink

assessmentReport.AppendIssues(AssessmentIssue{
Category: UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY,
Type: issue.Type,
Name: issue.Name,
Impact: issue.Impact,
Description: issue.Reason,
ObjectType: issue.ObjectType,
ObjectName: issue.ObjectName,
SqlStatement: issue.SqlStatement,
DocsLink: issue.DocsLink,
MinimumVersionFixedIn: issue.MinimumVersionsFixedIn,
Category: UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY,
Type: issue.Type,
Name: issue.Name,
Impact: issue.Impact,
Description: issue.Reason,
ObjectType: issue.ObjectType,
ObjectName: issue.ObjectName,
SqlStatement: issue.SqlStatement,
DocsLink: issue.DocsLink,
MinimumVersionsFixedIn: issue.MinimumVersionsFixedIn,
})
}
feature := UnsupportedFeature{
Expand Down Expand Up @@ -1335,14 +1305,14 @@ func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error
result = append(result, uqc)

assessmentReport.AppendIssues(AssessmentIssue{
Category: UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY,
Type: issue.Type,
Name: issue.Name,
Impact: issue.Impact,
Description: issue.Description,
SqlStatement: issue.SqlStatement,
DocsLink: issue.DocsLink,
MinimumVersionFixedIn: issue.MinimumVersionsFixedIn,
Category: UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY,
Type: issue.Type,
Name: issue.Name,
Impact: issue.Impact,
Description: issue.Description,
SqlStatement: issue.SqlStatement,
DocsLink: issue.DocsLink,
MinimumVersionsFixedIn: issue.MinimumVersionsFixedIn,
})
}
}
Expand Down Expand Up @@ -1441,16 +1411,16 @@ func getAssessmentIssuesForUnsupportedDatatypes(unsupportedDatatypes []utils.Tab
for _, colInfo := range unsupportedDatatypes {
qualifiedColName := fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName)
issue := AssessmentIssue{
Category: UNSUPPORTED_DATATYPES_CATEGORY,
CategoryDescription: GetCategoryDescription(UNSUPPORTED_DATATYPES_CATEGORY),
Type: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry"
Name: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry"
Description: "", // TODO
Impact: constants.IMPACT_LEVEL_3,
ObjectType: constants.COLUMN,
ObjectName: qualifiedColName,
DocsLink: "", // TODO
MinimumVersionFixedIn: nil, // TODO
Category: UNSUPPORTED_DATATYPES_CATEGORY,
CategoryDescription: GetCategoryDescription(UNSUPPORTED_DATATYPES_CATEGORY),
Type: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry"
Name: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry"
Description: "", // TODO
Impact: constants.IMPACT_LEVEL_3,
ObjectType: constants.COLUMN,
ObjectName: qualifiedColName,
DocsLink: "", // TODO
MinimumVersionsFixedIn: nil, // TODO
}
assessmentIssues = append(assessmentIssues, issue)
}
Expand Down
27 changes: 14 additions & 13 deletions yb-voyager/cmd/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1073,17 +1073,17 @@ type AssessmentReport struct {

// Fields apart from Category, CategoryDescription, TypeName and Impact will be populated only if/when available
type AssessmentIssue struct {
Category string `json:"Category"` // expected values: unsupported_features, unsupported_query_constructs, migration_caveats, unsupported_plpgsql_objects, unsupported_datatype
CategoryDescription string `json:"CategoryDescription"`
Type string `json:"Type"` // Ex: GIN_INDEXES, SECURITY_INVOKER_VIEWS, STORED_GENERATED_COLUMNS
Name string `json:"Name"` // Ex: GIN Indexes, Security Invoker Views, Stored Generated Columns
Description string `json:"Description"`
Impact string `json:"Impact"` // // Level-1, Level-2, Level-3 (no default: need to be assigned for each issue)
ObjectType string `json:"ObjectType"` // For datatype category, ObjectType will be datatype (for eg "geometry")
ObjectName string `json:"ObjectName"`
SqlStatement string `json:"SqlStatement"`
DocsLink string `json:"DocsLink"`
MinimumVersionFixedIn map[string]*ybversion.YBVersion `json:"MinimumVersionFixedIn"`
Category string `json:"Category"` // expected values: unsupported_features, unsupported_query_constructs, migration_caveats, unsupported_plpgsql_objects, unsupported_datatype
CategoryDescription string `json:"CategoryDescription"`
Type string `json:"Type"` // Ex: GIN_INDEXES, SECURITY_INVOKER_VIEWS, STORED_GENERATED_COLUMNS
Name string `json:"Name"` // Ex: GIN Indexes, Security Invoker Views, Stored Generated Columns
Description string `json:"Description"`
Impact string `json:"Impact"` // // Level-1, Level-2, Level-3 (no default: need to be assigned for each issue)
ObjectType string `json:"ObjectType"` // For datatype category, ObjectType will be datatype (for eg "geometry")
ObjectName string `json:"ObjectName"`
SqlStatement string `json:"SqlStatement"`
DocsLink string `json:"DocsLink"`
MinimumVersionsFixedIn map[string]*ybversion.YBVersion `json:"MinimumVersionsFixedIn"`
}

type UnsupportedFeature struct {
Expand Down Expand Up @@ -1128,6 +1128,9 @@ type AssessMigrationDBConfig struct {

// =============== for yugabyted controlplane ==============//
// TODO: see if this can be accommodated in controlplane pkg, facing pkg cyclic dependency issue

var ASSESS_MIGRATION_YBD_PAYLOAD_VERSION = "1.1" // version(s) till now: 1.0, 1.1

type AssessMigrationPayload struct {
PayloadVersion string
VoyagerVersion string
Expand Down Expand Up @@ -1194,8 +1197,6 @@ type TargetSizingRecommendations struct {
TotalShardedSize int64
}

var ASSESS_MIGRATION_PAYLOAD_VERSION = "1.1"

//====== AssesmentReport struct methods ======//

func ParseJSONToAssessmentReport(reportPath string) (*AssessmentReport, error) {
Expand Down
Loading

0 comments on commit ae842f5

Please sign in to comment.