Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: patch data migration scripts for non-MySQL databases #5297

Merged
merged 1 commit into from
Nov 26, 2024
Merged

fix: patch data migration scripts for non-MySQL databases #5297

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 65 additions & 17 deletions engine/apps/alerts/migrations/0063_migrate_channelfilter_slack_channel_id.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,71 @@ def populate_slack_channel(apps, schema_editor):

logger.info("Starting migration to populate slack_channel field.")

sql = f"""
UPDATE {ChannelFilter._meta.db_table} AS cf
JOIN {AlertReceiveChannel._meta.db_table} AS arc ON arc.id = cf.alert_receive_channel_id
JOIN {Organization._meta.db_table} AS org ON org.id = arc.organization_id
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = cf._slack_channel_id
AND sc.slack_team_identity_id = org.slack_team_identity_id
SET cf.slack_channel_id = sc.id
WHERE cf._slack_channel_id IS NOT NULL
AND org.slack_team_identity_id IS NOT NULL;
"""

with schema_editor.connection.cursor() as cursor:
cursor.execute(sql)
updated_rows = cursor.rowcount # Number of rows updated

logger.info(f"Bulk updated {updated_rows} ChannelFilters with their Slack channel.")
logger.info("Finished migration to populate slack_channel field.")
# NOTE: the following raw SQL only works on mysql, fall back to the less-efficient (but working) ORM method
# for non-mysql databases
#
# see the following references for more information:
# https://github.com/grafana/oncall/issues/5244#issuecomment-2493688544
# https://github.com/grafana/oncall/pull/5233/files#diff-d03cd69968936ddd363cb81aee15a643e4058d1e34bb191a407a0b8fe5fe0a77
if schema_editor.connection.vendor == "mysql":
sql = f"""
UPDATE {ChannelFilter._meta.db_table} AS cf
JOIN {AlertReceiveChannel._meta.db_table} AS arc ON arc.id = cf.alert_receive_channel_id
JOIN {Organization._meta.db_table} AS org ON org.id = arc.organization_id
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = cf._slack_channel_id
AND sc.slack_team_identity_id = org.slack_team_identity_id
SET cf.slack_channel_id = sc.id
WHERE cf._slack_channel_id IS NOT NULL
AND org.slack_team_identity_id IS NOT NULL;
"""

with schema_editor.connection.cursor() as cursor:
cursor.execute(sql)
updated_rows = cursor.rowcount # Number of rows updated

logger.info(f"Bulk updated {updated_rows} ChannelFilters with their Slack channel.")
logger.info("Finished migration to populate slack_channel field.")
else:
queryset = ChannelFilter.objects.filter(
_slack_channel_id__isnull=False,
alert_receive_channel__organization__slack_team_identity__isnull=False,
)
total_channel_filters = queryset.count()
updated_channel_filters = 0
missing_channel_filters = 0
channel_filters_to_update = []

logger.info(f"Total channel filters to process: {total_channel_filters}")

for channel_filter in queryset:
slack_id = channel_filter._slack_channel_id
slack_team_identity = channel_filter.alert_receive_channel.organization.slack_team_identity

try:
slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity)
channel_filter.slack_channel = slack_channel
channel_filters_to_update.append(channel_filter)

updated_channel_filters += 1
logger.info(
f"ChannelFilter {channel_filter.id} updated with SlackChannel {slack_channel.id} "
f"(slack_id: {slack_id})."
)
except SlackChannel.DoesNotExist:
missing_channel_filters += 1
logger.warning(
f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} "
f"does not exist for ChannelFilter {channel_filter.id}."
)

if channel_filters_to_update:
ChannelFilter.objects.bulk_update(channel_filters_to_update, ["slack_channel"])
logger.info(f"Bulk updated {len(channel_filters_to_update)} ChannelFilters with their Slack channel.")

logger.info(
f"Finished migration. Total channel filters processed: {total_channel_filters}. "
f"Channel filters updated: {updated_channel_filters}. Missing SlackChannels: {missing_channel_filters}."
)


class Migration(migrations.Migration):
Expand Down
87 changes: 69 additions & 18 deletions engine/apps/alerts/migrations/0064_migrate_resolutionnoteslackmessage_slack_channel_id.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,75 @@ def populate_slack_channel(apps, schema_editor):

logger.info("Starting migration to populate slack_channel field.")

sql = f"""
UPDATE {ResolutionNoteSlackMessage._meta.db_table} AS rsm
JOIN {AlertGroup._meta.db_table} AS ag ON ag.id = rsm.alert_group_id
JOIN {AlertReceiveChannel._meta.db_table} AS arc ON arc.id = ag.channel_id
JOIN {Organization._meta.db_table} AS org ON org.id = arc.organization_id
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = rsm._slack_channel_id
AND sc.slack_team_identity_id = org.slack_team_identity_id
SET rsm.slack_channel_id = sc.id
WHERE rsm._slack_channel_id IS NOT NULL
AND org.slack_team_identity_id IS NOT NULL;
"""

with schema_editor.connection.cursor() as cursor:
cursor.execute(sql)
updated_rows = cursor.rowcount # Number of rows updated

logger.info(f"Bulk updated {updated_rows} ResolutionNoteSlackMessage records with their Slack channel.")
logger.info("Finished migration to populate slack_channel field.")
# NOTE: the following raw SQL only works on mysql, fall back to the less-efficient (but working) ORM method
# for non-mysql databases
#
# see the following references for more information:
# https://github.com/grafana/oncall/issues/5244#issuecomment-2493688544
# https://github.com/grafana/oncall/pull/5233/files#diff-4ee42d7e773e6116d7c1d0280d2dbb053422ea55bfa5802a1f26ffbf23a28867
if schema_editor.connection.vendor == "mysql":
sql = f"""
UPDATE {ResolutionNoteSlackMessage._meta.db_table} AS rsm
JOIN {AlertGroup._meta.db_table} AS ag ON ag.id = rsm.alert_group_id
JOIN {AlertReceiveChannel._meta.db_table} AS arc ON arc.id = ag.channel_id
JOIN {Organization._meta.db_table} AS org ON org.id = arc.organization_id
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = rsm._slack_channel_id
AND sc.slack_team_identity_id = org.slack_team_identity_id
SET rsm.slack_channel_id = sc.id
WHERE rsm._slack_channel_id IS NOT NULL
AND org.slack_team_identity_id IS NOT NULL;
"""

with schema_editor.connection.cursor() as cursor:
cursor.execute(sql)
updated_rows = cursor.rowcount # Number of rows updated

logger.info(f"Bulk updated {updated_rows} ResolutionNoteSlackMessage records with their Slack channel.")
logger.info("Finished migration to populate slack_channel field.")
else:
queryset = ResolutionNoteSlackMessage.objects.filter(
_slack_channel_id__isnull=False,
alert_group__channel__organization__slack_team_identity__isnull=False,
)
total_resolution_notes = queryset.count()
updated_resolution_notes = 0
missing_resolution_notes = 0
resolution_notes_to_update = []

logger.info(f"Total resolution note slack messages to process: {total_resolution_notes}")

for resolution_note in queryset:
slack_id = resolution_note._slack_channel_id
slack_team_identity = resolution_note.alert_group.channel.organization.slack_team_identity

try:
slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity)
resolution_note.slack_channel = slack_channel
resolution_notes_to_update.append(resolution_note)

updated_resolution_notes += 1
logger.info(
f"ResolutionNoteSlackMessage {resolution_note.id} updated with SlackChannel {slack_channel.id} "
f"(slack_id: {slack_id})."
)
except SlackChannel.DoesNotExist:
missing_resolution_notes += 1
logger.warning(
f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} "
f"does not exist for ResolutionNoteSlackMessage {resolution_note.id}."
)

if resolution_notes_to_update:
ResolutionNoteSlackMessage.objects.bulk_update(resolution_notes_to_update, ["slack_channel"])
logger.info(
f"Bulk updated {len(resolution_notes_to_update)} ResolutionNoteSlackMessage with their Slack channel."
)

logger.info(
f"Finished migration. Total resolution note slack messages processed: {total_resolution_notes}. "
f"Resolution note slack messages updated: {updated_resolution_notes}. "
f"Missing SlackChannels: {missing_resolution_notes}."
)


class Migration(migrations.Migration):
Expand Down
77 changes: 61 additions & 16 deletions engine/apps/schedules/migrations/0019_auto_20241021_1735.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,22 +13,67 @@ def populate_slack_channel(apps, schema_editor):

logger.info("Starting migration to populate slack_channel field.")

sql = f"""
UPDATE {OnCallSchedule._meta.db_table} AS ocs
JOIN {Organization._meta.db_table} AS org ON org.id = ocs.organization_id
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = ocs.channel
AND sc.slack_team_identity_id = org.slack_team_identity_id
SET ocs.slack_channel_id = sc.id
WHERE ocs.channel IS NOT NULL
AND org.slack_team_identity_id IS NOT NULL;
"""

with schema_editor.connection.cursor() as cursor:
cursor.execute(sql)
updated_rows = cursor.rowcount # Number of rows updated

logger.info(f"Bulk updated {updated_rows} OnCallSchedules with their Slack channel.")
logger.info("Finished migration to populate slack_channel field.")
# NOTE: the following raw SQL only works on mysql, fall back to the less-efficient (but working) ORM method
# for non-mysql databases
#
# see the following references for more information:
# https://github.com/grafana/oncall/issues/5244#issuecomment-2493688544
# https://github.com/grafana/oncall/pull/5233/files#diff-d287631475456a42d005595383fb0b829cafb25aa15ed09b8e898b34803e59ef
if schema_editor.connection.vendor == "mysql":
sql = f"""
UPDATE {OnCallSchedule._meta.db_table} AS ocs
JOIN {Organization._meta.db_table} AS org ON org.id = ocs.organization_id
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = ocs.channel
AND sc.slack_team_identity_id = org.slack_team_identity_id
SET ocs.slack_channel_id = sc.id
WHERE ocs.channel IS NOT NULL
AND org.slack_team_identity_id IS NOT NULL;
"""

with schema_editor.connection.cursor() as cursor:
cursor.execute(sql)
updated_rows = cursor.rowcount # Number of rows updated

logger.info(f"Bulk updated {updated_rows} OnCallSchedules with their Slack channel.")
logger.info("Finished migration to populate slack_channel field.")
else:
queryset = OnCallSchedule.objects.filter(channel__isnull=False, organization__slack_team_identity__isnull=False)
total_schedules = queryset.count()
updated_schedules = 0
missing_channels = 0
schedules_to_update = []

logger.info(f"Total schedules to process: {total_schedules}")

for schedule in queryset:
slack_id = schedule.channel
slack_team_identity = schedule.organization.slack_team_identity

try:
slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity)

schedule.slack_channel = slack_channel
schedules_to_update.append(schedule)

updated_schedules += 1
logger.info(
f"Schedule {schedule.id} updated with SlackChannel {slack_channel.id} (slack_id: {slack_id})."
)
except SlackChannel.DoesNotExist:
missing_channels += 1
logger.warning(
f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} "
f"does not exist for Schedule {schedule.id}."
)

if schedules_to_update:
OnCallSchedule.objects.bulk_update(schedules_to_update, ["slack_channel"])
logger.info(f"Bulk updated {len(schedules_to_update)} OnCallSchedules with their Slack channel.")

logger.info(
f"Finished migration. Total schedules processed: {total_schedules}. "
f"Schedules updated: {updated_schedules}. Missing SlackChannels: {missing_channels}."
)

class Migration(migrations.Migration):

Expand Down
76 changes: 61 additions & 15 deletions engine/apps/user_management/migrations/0026_auto_20241017_1919.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,67 @@ def populate_default_slack_channel(apps, schema_editor):

logger.info("Starting migration to populate default_slack_channel field.")

sql = f"""
UPDATE {Organization._meta.db_table} AS org
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = org.general_log_channel_id
AND sc.slack_team_identity_id = org.slack_team_identity_id
SET org.default_slack_channel_id = sc.id
WHERE org.general_log_channel_id IS NOT NULL
AND org.slack_team_identity_id IS NOT NULL;
"""

with schema_editor.connection.cursor() as cursor:
cursor.execute(sql)
updated_rows = cursor.rowcount # Number of rows updated

logger.info(f"Bulk updated {updated_rows} organizations with their default Slack channel.")
logger.info("Finished migration to populate default_slack_channel field.")
# NOTE: the following raw SQL only works on mysql, fall back to the less-efficient (but working) ORM method
# for non-mysql databases
#
# see the following references for more information:
# https://github.com/grafana/oncall/issues/5244#issuecomment-2493688544
# https://github.com/grafana/oncall/pull/5233/files#diff-e69e0d7ecf51300be2ca5f4239c5f08b4c6e41de9856788f85a522001595a192
if schema_editor.connection.vendor == "mysql":
sql = f"""
UPDATE {Organization._meta.db_table} AS org
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = org.general_log_channel_id
AND sc.slack_team_identity_id = org.slack_team_identity_id
SET org.default_slack_channel_id = sc.id
WHERE org.general_log_channel_id IS NOT NULL
AND org.slack_team_identity_id IS NOT NULL;
"""

with schema_editor.connection.cursor() as cursor:
cursor.execute(sql)
updated_rows = cursor.rowcount # Number of rows updated

logger.info(f"Bulk updated {updated_rows} organizations with their default Slack channel.")
logger.info("Finished migration to populate default_slack_channel field.")
else:
queryset = Organization.objects.filter(general_log_channel_id__isnull=False, slack_team_identity__isnull=False)
total_orgs = queryset.count()
updated_orgs = 0
missing_channels = 0
organizations_to_update = []

logger.info(f"Total organizations to process: {total_orgs}")

for org in queryset:
slack_id = org.general_log_channel_id
slack_team_identity = org.slack_team_identity

try:
slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity)

org.default_slack_channel = slack_channel
organizations_to_update.append(org)

updated_orgs += 1
logger.info(
f"Organization {org.id} updated with SlackChannel {slack_channel.id} (slack_id: {slack_id})."
)
except SlackChannel.DoesNotExist:
missing_channels += 1
logger.warning(
f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} "
f"does not exist for Organization {org.id}."
)

if organizations_to_update:
Organization.objects.bulk_update(organizations_to_update, ["default_slack_channel"])
logger.info(f"Bulk updated {len(organizations_to_update)} organizations with their default Slack channel.")

logger.info(
f"Finished migration. Total organizations processed: {total_orgs}. "
f"Organizations updated: {updated_orgs}. Missing SlackChannels: {missing_channels}."
)


class Migration(migrations.Migration):

Expand Down
Loading