Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 103 additions & 12 deletions internal/diff/diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -2011,13 +2011,13 @@ func (d *ddlDiff) collectMigrationSQL(targetSchema string, collector *diffCollec
}

// First: Drop operations (in reverse dependency order)
d.generateDropSQL(targetSchema, collector, preDroppedViews)
typeDropsAfterTables := d.generateDropSQL(targetSchema, collector, preDroppedViews)

// Then: Create operations (in dependency order)
d.generateCreateSQL(targetSchema, collector)

// Finally: Modify operations
d.generateModifySQL(targetSchema, collector, preDroppedViews)
d.generateModifySQL(targetSchema, collector, preDroppedViews, typeDropsAfterTables)
}

// generatePreDropMaterializedViewsSQL drops materialized views that depend on
Expand Down Expand Up @@ -3085,7 +3085,7 @@ func (d *ddlDiff) generateCreateSQL(targetSchema string, collector *diffCollecto

// generateModifySQL generates ALTER statements
// preDroppedViews contains views that were already dropped in the pre-drop phase
func (d *ddlDiff) generateModifySQL(targetSchema string, collector *diffCollector, preDroppedViews map[string]bool) {
func (d *ddlDiff) generateModifySQL(targetSchema string, collector *diffCollector, preDroppedViews map[string]bool, typeDropsAfterTables []droppedTypeOrFunction) {
// Modify schemas
// Note: Schema modification is out of scope for schema-level comparisons

Expand All @@ -3105,6 +3105,10 @@ func (d *ddlDiff) generateModifySQL(targetSchema string, collector *diffCollecto
// Columns inheritance descendants drop once every parent has (see inherit.go)
generateInheritanceChildDropsSQL(d.inheritanceChildEffects, targetSchema, collector)

// Drop the types held back by generateDropSQL, now that the columns using
// them have been dropped or changed to another type (issue #710)
generateDropTypesAndFunctionsSQL(typeDropsAfterTables, nil, targetSchema, collector)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Reused type names block tables. The delayed drops run after every new table is created. If the old schema has enum priority and column tickets.priority, a valid desired schema can remove that column and enum and add table priority. The plan now tries to create table priority while the enum still owns that type name. PostgreSQL rejects the creation before the delayed drop can run. Defer creations that reuse these names until the columns and types have been removed.

Knowledge Base Used: Migration dependency ordering

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch — fixed in 84f8ecb. A dropped range is now also held for columns typed as its multirange (arrays included), and the drop_type case covers it.


// Attach OWNED BY for explicitly created sequences whose owning column was
// created by this migration, either with a new table (create phase) or by
// ALTER TABLE ... ADD COLUMN just above. The sequence itself was created
Expand Down Expand Up @@ -3207,7 +3211,11 @@ func (d *ddlDiff) generateModifySQL(targetSchema string, collector *diffCollecto

// generateDropSQL generates DROP statements in reverse dependency order
// preDroppedViews contains views that were already dropped in the pre-drop phase
func (d *ddlDiff) generateDropSQL(targetSchema string, collector *diffCollector, preDroppedViews map[string]bool) {
// generateDropSQL emits the drop phase. It returns the type drops (with the
// functions those types reference) that must wait for the modify phase,
// because a column of a modified table still uses the type until that table's
// ALTER TABLE has run (issue #710).
func (d *ddlDiff) generateDropSQL(targetSchema string, collector *diffCollector, preDroppedViews map[string]bool) []droppedTypeOrFunction {

// REVOKE privileges BEFORE dropping objects (objects must exist for REVOKE to succeed)
// Skip REVOKEs on relations already dropped in the pre-drop phase: REVOKE has
Expand Down Expand Up @@ -3297,7 +3305,32 @@ func (d *ddlDiff) generateDropSQL(targetSchema string, collector *diffCollector,
// Drop types, interleaved with the held-back functions in dependency
// order: a range before its subtype_diff function, that function before
// the subtype its signature takes, a domain before its CHECK function.
for _, node := range orderTypeAndFunctionDrops(droppedTypes, functionsAfterTypes) {
// A dropped type that a modified table's column still uses waits, along
// with whatever must be dropped after it, until that table has been
// altered (issue #710). The old definition of a replaced type cannot
// wait: its new definition is created in the create phase.
usedByColumns := typesUsedByAlteredColumns(d.droppedTypes, d.modifiedTables)
now, held := orderTypeAndFunctionDrops(droppedTypes, functionsAfterTypes, usedByColumns)
var afterTables []droppedTypeOrFunction
for _, node := range held {
if _, ok := replacedOld[node.typeObj]; ok && node.typeObj != nil {
now = append(now, node)
} else {
afterTables = append(afterTables, node)
Comment on lines +3315 to +3319

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Replaced ranges drop too early. Moving a replaced range back into now breaks the order established by orderTypeAndFunctionDrops. For example, an old domain slot can wrap range span, with a table column using slot. If the desired schema removes that column and domain while changing span's subtype, both types enter held, but this branch moves only span back. The plan drops span while slot still depends on it and fails. Keep the column removal, dependent drops, and range replacement in dependency order rather than moving the range alone.

Knowledge Base Used: Migration planning and diffing

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This scenario isn't made worse here: before this PR the domain was dropped ahead of the column and failed the same way. Ordering it correctly needs the old range dropped after the modify phase but before its replacement is created, which the current phase layout can't express. I'd rather leave that for a follow-up than widen this fix.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That’s a fair distinction. This dependency chain already failed before the PR because slot was dropped while the column still used it; the new ordering changes where it fails, but doesn’t introduce a newly failing scenario. Fixing it properly would require a phase that drops the old span after the table alteration and before creating its replacement, which is beyond this change’s scope. I’ll withdraw the comment and leave it for a follow-up.

Tip: You can customize Greptile's behavior for this repo with .greptile/rules.md and .greptile/config.json.

}
}
generateDropTypesAndFunctionsSQL(now, replacedOld, targetSchema, collector)

// Drop schemas
// Note: Schema deletion is out of scope for schema-level comparisons

return afterTables
}

// generateDropTypesAndFunctionsSQL emits the drops in the given order. The
// old definition of a replaced type is reported against its diff.
func generateDropTypesAndFunctionsSQL(nodes []droppedTypeOrFunction, replacedOld map[*ir.Type]*typeDiff, targetSchema string, collector *diffCollector) {
for _, node := range nodes {
if node.function != nil {
generateDropFunctionsSQL([]*ir.Function{node.function}, targetSchema, collector)
continue
Expand All @@ -3308,9 +3341,41 @@ func (d *ddlDiff) generateDropSQL(targetSchema string, collector *diffCollector,
}
generateDropTypeSQL(node.typeObj, source, targetSchema, collector)
}
}

// Drop schemas
// Note: Schema deletion is out of scope for schema-level comparisons
// typesUsedByAlteredColumns returns the dropped types that a column of a
// modified table uses in the current state: a column the migration drops, or
// one it changes to another type. Such a type can only be dropped once the
// table has been altered.
func typesUsedByAlteredColumns(droppedTypes []*ir.Type, modifiedTables []*tableDiff) map[*ir.Type]bool {
if len(droppedTypes) == 0 || len(modifiedTables) == 0 {
return nil
}
byKey := make(map[string]*ir.Type, len(droppedTypes))
for _, t := range droppedTypes {
byKey[typeGraphKey(t.Schema, t.Name)] = t
// A column typed as a range's multirange depends on the range.
if t.Kind == ir.TypeKindRange && t.MultirangeTypeName != "" {
if key := extractTypeName(t.MultirangeTypeName, t.Schema); key != "" {
byKey[key] = t
}
}
}
used := make(map[*ir.Type]bool)
mark := func(table *ir.Table, column *ir.Column) {
if t, ok := byKey[extractTypeName(column.DataType, table.Schema)]; ok {
Comment on lines +3354 to +3366

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Multirange columns still block drops. byKey contains only each range's name, but the inspector stores multirange columns under MultirangeTypeName. For example, removing a period_multirange column and its owning range period never marks period as held. The plan still emits DROP TYPE period before ALTER TABLE ... DROP COLUMN, causing the dependency error this PR fixes. Map multirange names, including their array forms, to their owning range before looking up columns.

Knowledge Base Used: Migration dependency ordering

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here: this combination already failed before the PR (the type drop ran before the column drop). Fixing it means deferring creates that reuse a dropped type's name, which is out of scope for this fix. Happy to open a follow-up issue.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That makes sense. This combination already failed before this PR, and handling creates that reuse a dropped type’s name would require broader phase reordering, so I agree it’s out of scope here. The updated lookup does cover multirange names (including arrays); I’ll withdraw this finding and leave the create-ordering case for a follow-up.

used[t] = true
}
}
for _, td := range modifiedTables {
for _, column := range td.DroppedColumns {
mark(td.Table, column)
}
for _, columnDiff := range td.ModifiedColumns {
mark(td.Table, columnDiff.Old)
}
}
return used
}

// filterPreDroppedViews returns views that haven't been pre-dropped
Expand Down Expand Up @@ -4756,7 +4821,10 @@ type droppedTypeOrFunction struct {
// signature uses (that subtype_diff takes the subtype), and a caller before
// its callee. Ties break with types first, then by name, so the output stays
// deterministic.
func orderTypeAndFunctionDrops(types []*ir.Type, functions []*ir.Function) []droppedTypeOrFunction {
//
// The held types, and everything that has to be dropped after one of them,
// are returned separately (in the same order) for the caller to emit later.
func orderTypeAndFunctionDrops(types []*ir.Type, functions []*ir.Function, held map[*ir.Type]bool) (now, later []droppedTypeOrFunction) {
typeMap, _, typeEdges := buildTypeGraph(types)
typeKey := func(t *ir.Type) string { return "t\x00" + typeGraphKey(t.Schema, t.Name) }
fnKey := func(f *ir.Function) string { return "f\x00" + f.Schema + "." + f.Name + "(" + f.GetArguments() + ")" }
Expand Down Expand Up @@ -4817,16 +4885,35 @@ func orderTypeAndFunctionDrops(types []*ir.Type, functions []*ir.Function) []dro
}
}

// Holding a node holds everything that must be dropped after it.
heldKeys := make(map[string]bool)
var queue []string
for _, t := range types {
if held[t] {
heldKeys[typeKey(t)] = true
queue = append(queue, typeKey(t))
}
}
for len(queue) > 0 {
current := queue[0]
queue = queue[1:]
for _, next := range after[current] {
if _, exists := nodes[next]; exists && !heldKeys[next] {
heldKeys[next] = true
queue = append(queue, next)
}
}
}

var ready []string
for key := range nodes {
if inDegree[key] == 0 {
ready = append(ready, key)
}
}
sort.Strings(ready) // "f\x00" sorts before "t\x00"; reversed below so types lead
var result []droppedTypeOrFunction
done := make(map[string]bool, len(nodes))
for len(result) < len(nodes) {
for len(done) < len(nodes) {
if len(ready) == 0 {
// A cycle cannot occur in PostgreSQL's catalog; keep going
// deterministically rather than drop nodes on the floor.
Expand All @@ -4845,7 +4932,11 @@ func orderTypeAndFunctionDrops(types []*ir.Type, functions []*ir.Function) []dro
continue
}
done[current] = true
result = append(result, nodes[current])
if heldKeys[current] {
later = append(later, nodes[current])
} else {
now = append(now, nodes[current])
}
for _, next := range after[current] {
inDegree[next]--
if inDegree[next] <= 0 && !done[next] {
Expand All @@ -4854,7 +4945,7 @@ func orderTypeAndFunctionDrops(types []*ir.Type, functions []*ir.Function) []dro
}
}
}
return result
return now, later
}

// typeReferencesNewFunction determines if a domain references any newly added
Expand Down
31 changes: 27 additions & 4 deletions internal/diff/type_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,8 @@ func TestOrderTypeAndFunctionDrops(t *testing.T) {
spanSet := &ir.Type{Schema: "public", Name: "busy", Kind: ir.TypeKindDomain, BaseType: "public.positive_span_multirange"}

var names []string
for _, node := range orderTypeAndFunctionDrops([]*ir.Type{positive, span, spanSet}, []*ir.Function{isPositive, positiveDiff}) {
nodes, _ := orderTypeAndFunctionDrops([]*ir.Type{positive, span, spanSet}, []*ir.Function{isPositive, positiveDiff}, nil)
for _, node := range nodes {
if node.function != nil {
names = append(names, node.function.Name)
} else {
Expand All @@ -232,7 +233,8 @@ func TestOrderTypeAndFunctionDrops(t *testing.T) {
bodyFn := &ir.Function{Schema: "public", Name: "span_diff", TypeDependencies: []string{"public.z_status"}, Parameters: []*ir.Parameter{{Mode: "IN", DataType: "integer"}, {Mode: "IN", DataType: "integer"}}}
intSpan := &ir.Type{Schema: "public", Name: "span", Kind: ir.TypeKindRange, Subtype: "integer", SubtypeDiff: "span_diff", MultirangeTypeName: "span_multirange"}
names = nil
for _, node := range orderTypeAndFunctionDrops([]*ir.Type{status, intSpan}, []*ir.Function{bodyFn}) {
nodes, _ = orderTypeAndFunctionDrops([]*ir.Type{status, intSpan}, []*ir.Function{bodyFn}, nil)
for _, node := range nodes {
if node.function != nil {
names = append(names, node.function.Name)
} else {
Expand All @@ -251,7 +253,8 @@ func TestOrderTypeAndFunctionDrops(t *testing.T) {
aSpan := &ir.Type{Schema: "public", Name: "a_span", Kind: ir.TypeKindRange, Subtype: "integer", SubtypeDiff: "a_diff", MultirangeTypeName: "a_span_multirange"}
zSpan := &ir.Type{Schema: "public", Name: "z_span", Kind: ir.TypeKindRange, Subtype: "integer", SubtypeDiff: "z_diff", MultirangeTypeName: "z_span_multirange"}
names = nil
for _, node := range orderTypeAndFunctionDrops([]*ir.Type{aSpan, zSpan}, []*ir.Function{aDiff, zDiff}) {
nodes, _ = orderTypeAndFunctionDrops([]*ir.Type{aSpan, zSpan}, []*ir.Function{aDiff, zDiff}, nil)
for _, node := range nodes {
if node.function != nil {
names = append(names, node.function.Name)
} else {
Expand All @@ -264,12 +267,32 @@ func TestOrderTypeAndFunctionDrops(t *testing.T) {

// No held functions: plain reverse dependency order of the types.
names = nil
for _, node := range orderTypeAndFunctionDrops([]*ir.Type{positive, span}, nil) {
nodes, _ = orderTypeAndFunctionDrops([]*ir.Type{positive, span}, nil, nil)
for _, node := range nodes {
names = append(names, node.typeObj.Name)
}
if strings.Join(names, ",") != "positive_span,positive" {
t.Fatalf("got %v", names)
}

// A type a modified table still uses is held back together with all
// that must be dropped after it; what depends on it still goes first.
names = nil
nodes, later := orderTypeAndFunctionDrops([]*ir.Type{positive, span, spanSet}, []*ir.Function{isPositive, positiveDiff}, map[*ir.Type]bool{span: true})
for _, node := range nodes {
names = append(names, node.typeObj.Name)
}
names = append(names, "|")
for _, node := range later {
if node.function != nil {
names = append(names, node.function.Name)
} else {
names = append(names, node.typeObj.Name)
}
}
if strings.Join(names, ",") != "busy,|,positive_span,positive_diff,positive,is_positive" {
t.Fatalf("got %v", names)
}
}

func TestGenerateAlterCompositeStatements(t *testing.T) {
Expand Down
10 changes: 10 additions & 0 deletions testdata/diff/create_type/drop_type/diff.sql
Original file line number Diff line number Diff line change
@@ -1 +1,11 @@
ALTER TABLE tickets DROP COLUMN priority;

ALTER TABLE tickets DROP COLUMN busy;

ALTER TABLE tickets ALTER COLUMN severity TYPE text USING severity::text;

DROP TYPE IF EXISTS severity RESTRICT;

DROP TYPE IF EXISTS priority RESTRICT;

DROP TYPE IF EXISTS period RESTRICT;
7 changes: 6 additions & 1 deletion testdata/diff/create_type/drop_type/new.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,9 @@ CREATE TYPE public.status AS ENUM (
'active',
'inactive',
'pending'
);
);

CREATE TABLE public.tickets (
id integer PRIMARY KEY,
severity text
);
21 changes: 20 additions & 1 deletion testdata/diff/create_type/drop_type/old.sql
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,23 @@ CREATE TYPE public.priority AS ENUM (
'medium',
'high',
'urgent'
);
);

CREATE TYPE public.severity AS ENUM (
'minor',
'major'
);

CREATE TYPE public.period AS RANGE (
subtype = date
);

-- Columns using the dropped types go away, or change type, in the same
-- migration: they must be altered before DROP TYPE (issue #710). A multirange
-- column holds its range the same way.
CREATE TABLE public.tickets (
id integer PRIMARY KEY,
priority public.priority,
severity public.severity,
busy public.period_multirange[]
);
32 changes: 31 additions & 1 deletion testdata/diff/create_type/drop_type/plan.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,46 @@
"pgschema_version": "1.14.1",
"created_at": "1970-01-01T00:00:00Z",
"source_fingerprint": {
"hash": "7739601f0084f192ccf2d01e2d6602820b2f4e39b5acdd7c4dd14c6d3b8a3da1"
"hash": "4e611f2c250cecd0eddab3611e85d8e7261c5c9d4ecae300961895c3e76e4fb8"
},
"groups": [
{
"steps": [
{
"sql": "ALTER TABLE tickets DROP COLUMN priority;",
"type": "table.column",
"operation": "drop",
"path": "public.tickets.priority"
},
{
"sql": "ALTER TABLE tickets DROP COLUMN busy;",
"type": "table.column",
"operation": "drop",
"path": "public.tickets.busy"
},
{
"sql": "ALTER TABLE tickets ALTER COLUMN severity TYPE text USING severity::text;",
"type": "table.column",
"operation": "alter",
"path": "public.tickets.severity"
},
{
"sql": "DROP TYPE IF EXISTS severity RESTRICT;",
"type": "type",
"operation": "drop",
"path": "public.severity"
},
{
"sql": "DROP TYPE IF EXISTS priority RESTRICT;",
"type": "type",
"operation": "drop",
"path": "public.priority"
},
{
"sql": "DROP TYPE IF EXISTS period RESTRICT;",
"type": "type",
"operation": "drop",
"path": "public.period"
}
]
}
Expand Down
10 changes: 10 additions & 0 deletions testdata/diff/create_type/drop_type/plan.sql
Original file line number Diff line number Diff line change
@@ -1 +1,11 @@
ALTER TABLE tickets DROP COLUMN priority;

ALTER TABLE tickets DROP COLUMN busy;

ALTER TABLE tickets ALTER COLUMN severity TYPE text USING severity::text;

DROP TYPE IF EXISTS severity RESTRICT;

DROP TYPE IF EXISTS priority RESTRICT;

DROP TYPE IF EXISTS period RESTRICT;
Loading