-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for listening to Azure Maintenance Events (#1876)
Adding an automatic subscription to the AzureRedisEvents pubsub channel for Azure caches. This channel notifies clients of upcoming maintenance and failover events. By exposing these events, users will be able to know about maintenance ahead of time, and can implement their own logic (e.g. diverting traffic from the cache to another database for the duration of the maintenance event) in response, with the goal of minimizing downtime and disrupted connections. We also automatically refresh our view of the topology of the cluster in response to certain events. Here are some of the possible notifications: ``` // Indicates that a maintenance event is scheduled. May be several minutes from now NodeMaintenanceScheduled, // This event gets fired ~20s before maintenance begins NodeMaintenanceStarting, // This event gets fired when maintenance is imminent (<5s) NodeMaintenanceStart, // Indicates that the node maintenance operation is over NodeMaintenanceEnded, // Indicates that a replica has been promoted to primary NodeMaintenanceFailover ``` Co-authored-by: Michelle Soedal <[email protected]> Co-authored-by: Nick Craver <[email protected]>
- Loading branch information
3 people
committed
Oct 7, 2021
1 parent
082c69b
commit ef1178e
Showing
9 changed files
with
414 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
194 changes: 194 additions & 0 deletions
194
src/StackExchange.Redis/Maintenance/AzureMaintenanceEvent.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,194 @@ | ||
| using System; | ||
| using System.Globalization; | ||
| using System.Net; | ||
| using System.Threading.Tasks; | ||
| #if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER | ||
| using System.Buffers.Text; | ||
| #endif | ||
|
|
||
| namespace StackExchange.Redis.Maintenance | ||
| { | ||
| /// <summary> | ||
| /// Azure node maintenance event. For more information, please see: https://aka.ms/redis/maintenanceevents | ||
| /// </summary> | ||
| public sealed class AzureMaintenanceEvent : ServerMaintenanceEvent | ||
| { | ||
| private const string PubSubChannelName = "AzureRedisEvents"; | ||
|
|
||
| internal AzureMaintenanceEvent(string azureEvent) | ||
| { | ||
| if (azureEvent == null) | ||
| { | ||
| return; | ||
| } | ||
|
|
||
| // The message consists of key-value pairs delimited by pipes. For example, a message might look like: | ||
| // NotificationType|NodeMaintenanceStarting|StartTimeUtc|2021-09-23T12:34:19|IsReplica|False|IpAddress|13.67.42.199|SSLPort|15001|NonSSLPort|13001 | ||
| var message = azureEvent.AsSpan(); | ||
| try | ||
| { | ||
| while (message.Length > 0) | ||
| { | ||
| if (message[0] == '|') | ||
| { | ||
| message = message.Slice(1); | ||
| continue; | ||
| } | ||
|
|
||
| // Grab the next pair | ||
| var nextDelimiter = message.IndexOf('|'); | ||
| if (nextDelimiter < 0) | ||
| { | ||
| // The rest of the message is not a key-value pair and is therefore malformed. Stop processing it. | ||
| break; | ||
| } | ||
|
|
||
| if (nextDelimiter == message.Length - 1) | ||
| { | ||
| // The message is missing the value for this key-value pair. It is malformed so we stop processing it. | ||
| break; | ||
| } | ||
|
|
||
| var key = message.Slice(0, nextDelimiter); | ||
| message = message.Slice(key.Length + 1); | ||
|
|
||
| var valueEnd = message.IndexOf('|'); | ||
| var value = valueEnd > -1 ? message.Slice(0, valueEnd) : message; | ||
| message = message.Slice(value.Length); | ||
|
|
||
| if (key.Length > 0 && value.Length > 0) | ||
| { | ||
| #if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER | ||
| switch (key) | ||
| { | ||
| case var _ when key.SequenceEqual(nameof(NotificationType).AsSpan()): | ||
| NotificationTypeString = value.ToString(); | ||
| NotificationType = ParseNotificationType(NotificationTypeString); | ||
| break; | ||
| case var _ when key.SequenceEqual("StartTimeInUTC".AsSpan()) && DateTime.TryParseExact(value, "s", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTime startTime): | ||
| StartTimeUtc = DateTime.SpecifyKind(startTime, DateTimeKind.Utc); | ||
| break; | ||
| case var _ when key.SequenceEqual(nameof(IsReplica).AsSpan()) && bool.TryParse(value, out var isReplica): | ||
| IsReplica = isReplica; | ||
| break; | ||
| case var _ when key.SequenceEqual(nameof(IPAddress).AsSpan()) && IPAddress.TryParse(value, out var ipAddress): | ||
| IPAddress = ipAddress; | ||
| break; | ||
| case var _ when key.SequenceEqual("SSLPort".AsSpan()) && Format.TryParseInt32(value, out var port): | ||
| SslPort = port; | ||
| break; | ||
| case var _ when key.SequenceEqual("NonSSLPort".AsSpan()) && Format.TryParseInt32(value, out var nonsslport): | ||
| NonSslPort = nonsslport; | ||
| break; | ||
| default: | ||
| break; | ||
| } | ||
| #else | ||
| switch (key) | ||
| { | ||
| case var _ when key.SequenceEqual(nameof(NotificationType).AsSpan()): | ||
| NotificationTypeString = value.ToString(); | ||
| NotificationType = ParseNotificationType(NotificationTypeString); | ||
| break; | ||
| case var _ when key.SequenceEqual("StartTimeInUTC".AsSpan()) && DateTime.TryParseExact(value.ToString(), "s", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTime startTime): | ||
| StartTimeUtc = DateTime.SpecifyKind(startTime, DateTimeKind.Utc); | ||
| break; | ||
| case var _ when key.SequenceEqual(nameof(IsReplica).AsSpan()) && bool.TryParse(value.ToString(), out var isReplica): | ||
| IsReplica = isReplica; | ||
| break; | ||
| case var _ when key.SequenceEqual(nameof(IPAddress).AsSpan()) && IPAddress.TryParse(value.ToString(), out var ipAddress): | ||
| IPAddress = ipAddress; | ||
| break; | ||
| case var _ when key.SequenceEqual("SSLPort".AsSpan()) && Format.TryParseInt32(value.ToString(), out var port): | ||
| SslPort = port; | ||
| break; | ||
| case var _ when key.SequenceEqual("NonSSLPort".AsSpan()) && Format.TryParseInt32(value.ToString(), out var nonsslport): | ||
| NonSslPort = nonsslport; | ||
| break; | ||
| default: | ||
| break; | ||
| } | ||
| #endif | ||
| } | ||
| } | ||
| } | ||
| catch | ||
| { | ||
| // TODO: Append to rolling debug log when it's present | ||
| } | ||
| } | ||
|
|
||
| internal async static Task AddListenerAsync(ConnectionMultiplexer multiplexer, ConnectionMultiplexer.LogProxy logProxy) | ||
| { | ||
| try | ||
| { | ||
| var sub = multiplexer.GetSubscriber(); | ||
| if (sub == null) | ||
| { | ||
| logProxy?.WriteLine("Failed to GetSubscriber for AzureRedisEvents"); | ||
| return; | ||
| } | ||
|
|
||
| await sub.SubscribeAsync(PubSubChannelName, async (channel, message) => | ||
| { | ||
| var newMessage = new AzureMaintenanceEvent(message); | ||
| multiplexer.InvokeServerMaintenanceEvent(newMessage); | ||
| switch (newMessage.NotificationType) | ||
| { | ||
| case AzureNotificationType.NodeMaintenanceEnded: | ||
| case AzureNotificationType.NodeMaintenanceFailoverComplete: | ||
| await multiplexer.ReconfigureAsync(first: false, reconfigureAll: true, log: logProxy, blame: null, cause: $"Azure Event: {newMessage.NotificationType}").ForAwait(); | ||
| break; | ||
| } | ||
| }).ForAwait(); | ||
| } | ||
| catch (Exception e) | ||
| { | ||
| logProxy?.WriteLine($"Encountered exception: {e}"); | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Indicates the type of event (raw string form). | ||
| /// </summary> | ||
| public string NotificationTypeString { get; } | ||
|
|
||
| /// <summary> | ||
| /// The parsed version of <see cref="NotificationTypeString"/> for easier consumption. | ||
| /// </summary> | ||
| public AzureNotificationType NotificationType { get; } | ||
|
|
||
| /// <summary> | ||
| /// Indicates if the event is for a replica node. | ||
| /// </summary> | ||
| public bool IsReplica { get; } | ||
|
|
||
| /// <summary> | ||
| /// IPAddress of the node event is intended for. | ||
| /// </summary> | ||
| public IPAddress IPAddress { get; } | ||
|
|
||
| /// <summary> | ||
| /// SSL Port. | ||
| /// </summary> | ||
| public int SslPort { get; } | ||
|
|
||
| /// <summary> | ||
| /// Non-SSL port. | ||
| /// </summary> | ||
| public int NonSslPort { get; } | ||
|
|
||
| private AzureNotificationType ParseNotificationType(string typeString) => typeString switch | ||
| { | ||
| "NodeMaintenanceScheduled" => AzureNotificationType.NodeMaintenanceScheduled, | ||
| "NodeMaintenanceStarting" => AzureNotificationType.NodeMaintenanceStarting, | ||
| "NodeMaintenanceStart" => AzureNotificationType.NodeMaintenanceStart, | ||
| "NodeMaintenanceEnded" => AzureNotificationType.NodeMaintenanceEnded, | ||
| // This is temporary until server changes go into effect - to be removed in later versions | ||
| "NodeMaintenanceFailover" => AzureNotificationType.NodeMaintenanceFailoverComplete, | ||
| "NodeMaintenanceFailoverComplete" => AzureNotificationType.NodeMaintenanceFailoverComplete, | ||
| _ => AzureNotificationType.Unknown, | ||
| }; | ||
| } | ||
| } |
38 changes: 38 additions & 0 deletions
38
src/StackExchange.Redis/Maintenance/AzureNotificationType.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| namespace StackExchange.Redis.Maintenance | ||
| { | ||
| /// <summary> | ||
| /// The types of notifications that Azure is sending for events happening. | ||
| /// </summary> | ||
| public enum AzureNotificationType | ||
| { | ||
| /// <summary> | ||
| /// Unrecognized event type, likely needs a library update to recognize new events. | ||
| /// </summary> | ||
| Unknown, | ||
|
|
||
| /// <summary> | ||
| /// Indicates that a maintenance event is scheduled. May be several minutes from now. | ||
| /// </summary> | ||
| NodeMaintenanceScheduled, | ||
|
|
||
| /// <summary> | ||
| /// This event gets fired ~20s before maintenance begins. | ||
| /// </summary> | ||
| NodeMaintenanceStarting, | ||
|
|
||
| /// <summary> | ||
| /// This event gets fired when maintenance is imminent (<5s). | ||
| /// </summary> | ||
| NodeMaintenanceStart, | ||
|
|
||
| /// <summary> | ||
| /// Indicates that the node maintenance operation is over. | ||
| /// </summary> | ||
| NodeMaintenanceEnded, | ||
|
|
||
| /// <summary> | ||
| /// Indicates that a replica has been promoted to primary. | ||
| /// </summary> | ||
| NodeMaintenanceFailoverComplete, | ||
| } | ||
| } |
Oops, something went wrong.