Implement iOS VoiceOver support #18016
+388
−23
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,270 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using Avalonia.Automation; | ||
| using Avalonia.Automation.Peers; | ||
| using Avalonia.Automation.Provider; | ||
| using CoreGraphics; | ||
| using Foundation; | ||
| using UIKit; | ||
|
|
||
| namespace Avalonia.iOS | ||
| { | ||
| internal class AutomationPeerWrapper : UIAccessibilityElement | ||
| { | ||
| private static readonly IReadOnlyDictionary<AutomationProperty, Action<AutomationPeerWrapper>> s_propertySetters = | ||
| new Dictionary<AutomationProperty, Action<AutomationPeerWrapper>>() | ||
| { | ||
| { AutomationElementIdentifiers.NameProperty, UpdateName }, | ||
| { AutomationElementIdentifiers.HelpTextProperty, UpdateHelpText }, | ||
| { AutomationElementIdentifiers.BoundingRectangleProperty, UpdateBoundingRectangle }, | ||
|
|
||
| { RangeValuePatternIdentifiers.IsReadOnlyProperty, UpdateIsReadOnly }, | ||
| { RangeValuePatternIdentifiers.ValueProperty, UpdateValue }, | ||
|
|
||
| { ValuePatternIdentifiers.IsReadOnlyProperty, UpdateIsReadOnly }, | ||
| { ValuePatternIdentifiers.ValueProperty, UpdateValue }, | ||
| }; | ||
|
|
||
| private readonly AvaloniaView _view; | ||
| private readonly AutomationPeer _peer; | ||
|
|
||
| public AutomationPeerWrapper(AvaloniaView view, AutomationPeer? peer = null) : base(view) | ||
| { | ||
| _view = view; | ||
| _peer = peer ?? ControlAutomationPeer.CreatePeerForElement(view.TopLevel); | ||
|
|
||
| _peer.PropertyChanged += PeerPropertyChanged; | ||
| _peer.ChildrenChanged += PeerChildrenChanged; | ||
|
|
||
| AccessibilityContainer = _view; | ||
| AccessibilityIdentifier = _peer.GetAutomationId(); | ||
| } | ||
|
|
||
| private static void UpdateName(AutomationPeerWrapper self) | ||
| { | ||
| AutomationPeer peer = self; | ||
| self.AccessibilityLabel = peer.GetName(); | ||
| } | ||
|
|
||
| private static void UpdateHelpText(AutomationPeerWrapper self) | ||
| { | ||
| AutomationPeer peer = self; | ||
| self.AccessibilityHint = peer.GetHelpText(); | ||
| } | ||
|
|
||
| private static void UpdateBoundingRectangle(AutomationPeerWrapper self) | ||
| { | ||
| AutomationPeer peer = self; | ||
| Rect bounds = peer.GetBoundingRectangle(); | ||
| PixelRect screenRect = new PixelRect( | ||
| self._view.TopLevel.PointToScreen(bounds.TopLeft), | ||
| self._view.TopLevel.PointToScreen(bounds.BottomRight) | ||
| ); | ||
| self.AccessibilityFrame = new CGRect( | ||
| screenRect.X, screenRect.Y, | ||
| screenRect.Width, screenRect.Height | ||
| ); | ||
| } | ||
|
|
||
| private static void UpdateIsReadOnly(AutomationPeerWrapper self) | ||
| { | ||
| AutomationPeer peer = self; | ||
| self.AccessibilityRespondsToUserInteraction = | ||
| peer.GetProvider<IValueProvider>()?.IsReadOnly ?? | ||
| peer.GetProvider<IRangeValueProvider>()?.IsReadOnly ?? | ||
| peer.IsEnabled(); | ||
| } | ||
|
|
||
| private static void UpdateValue(AutomationPeerWrapper self) | ||
| { | ||
| AutomationPeer peer = self; | ||
| string? newValue = | ||
| peer.GetProvider<IRangeValueProvider>()?.Value.ToString("0.##") ?? | ||
| peer.GetProvider<IValueProvider>()?.Value; | ||
| if (self.AccessibilityValue != newValue) | ||
| { | ||
| self.AccessibilityValue = newValue; | ||
| UIAccessibility.PostNotification(UIAccessibilityPostNotification.Announcement, self); | ||
| } | ||
| } | ||
|
|
||
| private void PeerChildrenChanged(object? sender, EventArgs e) => _view.UpdateChildren(_peer); | ||
|
|
||
| private void PeerPropertyChanged(object? sender, AutomationPropertyChangedEventArgs e) => UpdateProperties(e.Property); | ||
|
|
||
| private void UpdateProperties(params AutomationProperty[] properties) | ||
| { | ||
| Action<AutomationPeerWrapper>? setter = | ||
| Delegate.Combine(properties | ||
| .Where(s_propertySetters.ContainsKey) | ||
| .Select(x => s_propertySetters[x]) | ||
| .Distinct() | ||
| .ToArray()) as Action<AutomationPeerWrapper>; | ||
| setter?.Invoke(this); | ||
IsaMorphic marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
| public bool UpdatePropertiesIfValid() | ||
| { | ||
| if (_peer.IsContentElement() && !_peer.IsOffscreen()) | ||
| { | ||
| UpdateProperties(s_propertySetters.Keys.ToArray()); | ||
| return IsAccessibilityElement = true; | ||
| } | ||
| else | ||
| { | ||
| return IsAccessibilityElement = false; | ||
| } | ||
| } | ||
|
|
||
| public void UpdateTraits() | ||
| { | ||
| UIAccessibilityTrait traits = UIAccessibilityTrait.None; | ||
|
|
||
| switch (_peer.GetAutomationControlType()) | ||
| { | ||
| case AutomationControlType.Button: | ||
| traits |= UIAccessibilityTrait.Button; | ||
| break; | ||
| case AutomationControlType.Header: | ||
| traits |= UIAccessibilityTrait.Header; | ||
| break; | ||
| case AutomationControlType.Hyperlink: | ||
| traits |= UIAccessibilityTrait.Link; | ||
| break; | ||
| case AutomationControlType.Image: | ||
| traits |= UIAccessibilityTrait.Image; | ||
| break; | ||
| } | ||
|
|
||
| if (_peer.GetProvider<IRangeValueProvider>()?.IsReadOnly == false) | ||
| { | ||
| traits |= UIAccessibilityTrait.Adjustable; | ||
| } | ||
|
|
||
| if (_peer.GetProvider<ISelectionItemProvider>()?.IsSelected == true) | ||
| { | ||
| traits |= UIAccessibilityTrait.Selected; | ||
| } | ||
|
|
||
| if (_peer.GetProvider<IValueProvider>()?.IsReadOnly == false) | ||
| { | ||
| traits |= UIAccessibilityTrait.UpdatesFrequently; | ||
| } | ||
|
|
||
| if (_peer.IsEnabled() == false) | ||
| { | ||
| traits |= UIAccessibilityTrait.NotEnabled; | ||
| } | ||
|
|
||
| AccessibilityTraits = (ulong)traits; | ||
| } | ||
|
|
||
| [Export("accessibilityActivate")] | ||
| public bool AccessibilityActivate() | ||
| { | ||
| IToggleProvider? toggleProvider = _peer.GetProvider<IToggleProvider>(); | ||
| IInvokeProvider? invokeProvider = _peer.GetProvider<IInvokeProvider>(); | ||
| if (toggleProvider is not null) | ||
| { | ||
| toggleProvider.Toggle(); | ||
| return true; | ||
| } | ||
| else if (invokeProvider is not null) | ||
| { | ||
| invokeProvider.Invoke(); | ||
| return true; | ||
| } | ||
| else | ||
| { | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| public override bool AccessibilityElementIsFocused() | ||
| { | ||
| base.AccessibilityElementIsFocused(); | ||
| return _peer.HasKeyboardFocus(); | ||
| } | ||
|
|
||
| public override void AccessibilityElementDidBecomeFocused() | ||
| { | ||
| base.AccessibilityElementDidBecomeFocused(); | ||
| _peer.BringIntoView(); | ||
| } | ||
|
|
||
| public override void AccessibilityDecrement() | ||
| { | ||
| base.AccessibilityDecrement(); | ||
| IRangeValueProvider? provider = _peer.GetProvider<IRangeValueProvider>(); | ||
| if (provider is not null) | ||
| { | ||
| double value = provider.Value; | ||
| provider.SetValue(value - provider.SmallChange); | ||
| } | ||
| } | ||
|
|
||
| public override void AccessibilityIncrement() | ||
| { | ||
| base.AccessibilityIncrement(); | ||
| IRangeValueProvider? provider = _peer.GetProvider<IRangeValueProvider>(); | ||
| if (provider is not null) | ||
| { | ||
| double value = provider.Value; | ||
| provider.SetValue(value + provider.SmallChange); | ||
| } | ||
| } | ||
|
|
||
| public override bool AccessibilityScroll(UIAccessibilityScrollDirection direction) | ||
| { | ||
| base.AccessibilityScroll(direction); | ||
| IScrollProvider? scrollProvider = _peer.GetProvider<IScrollProvider>(); | ||
| if (scrollProvider is not null) | ||
| { | ||
| bool didScroll; | ||
| ScrollAmount verticalAmount, horizontalAmount; | ||
| switch (direction) | ||
| { | ||
| case UIAccessibilityScrollDirection.Up: | ||
| verticalAmount = ScrollAmount.SmallIncrement; | ||
| horizontalAmount = ScrollAmount.NoAmount; | ||
| didScroll = true; | ||
| break; | ||
| case UIAccessibilityScrollDirection.Down: | ||
| verticalAmount = ScrollAmount.SmallDecrement; | ||
| horizontalAmount = ScrollAmount.NoAmount; | ||
| didScroll = true; | ||
| break; | ||
| case UIAccessibilityScrollDirection.Left: | ||
| verticalAmount = ScrollAmount.NoAmount; | ||
| horizontalAmount = ScrollAmount.SmallIncrement; | ||
| didScroll = true; | ||
| break; | ||
| case UIAccessibilityScrollDirection.Right: | ||
| verticalAmount = ScrollAmount.NoAmount; | ||
| horizontalAmount = ScrollAmount.SmallDecrement; | ||
| didScroll = true; | ||
| break; | ||
| default: | ||
| verticalAmount = ScrollAmount.NoAmount; | ||
| horizontalAmount = ScrollAmount.NoAmount; | ||
| didScroll = false; | ||
| break; | ||
| } | ||
|
|
||
| scrollProvider.Scroll(verticalAmount, horizontalAmount); | ||
| if (didScroll) | ||
| { | ||
| UIAccessibility.PostNotification(UIAccessibilityPostNotification.PageScrolled, this); | ||
| return true; | ||
| } | ||
| } | ||
| return false; | ||
| } | ||
|
|
||
| public static implicit operator AutomationPeer(AutomationPeerWrapper instance) | ||
| { | ||
| return instance._peer; | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using Avalonia.Automation.Peers; | ||
| using Foundation; | ||
| using UIKit; | ||
|
|
||
| namespace Avalonia.iOS | ||
| { | ||
| public partial class AvaloniaView : IUIAccessibilityContainer | ||
| { | ||
| private readonly List<AutomationPeer> _childrenList = new(); | ||
| private readonly Dictionary<AutomationPeer, AutomationPeerWrapper> _childrenMap = new(); | ||
|
|
||
| [Export("accessibilityContainerType")] | ||
| public UIAccessibilityContainerType AccessibilityContainerType { get; set; } = | ||
| UIAccessibilityContainerType.SemanticGroup; | ||
|
|
||
| [Export("accessibilityElementCount")] | ||
| public nint AccessibilityElementCount() | ||
| { | ||
| UpdateChildren(_accessWrapper); | ||
| return _childrenList.Count; | ||
| } | ||
|
|
||
| [Export("accessibilityElementAtIndex:")] | ||
| public NSObject? GetAccessibilityElementAt(nint index) | ||
| { | ||
| try | ||
| { | ||
| var wrapper = _childrenMap[_childrenList[(int)index]]; | ||
| if (wrapper.UpdatePropertiesIfValid()) | ||
| { | ||
| return wrapper; | ||
| } | ||
| else | ||
| { | ||
| _childrenList.Remove(wrapper); | ||
| _childrenMap.Remove(wrapper); | ||
| } | ||
| } | ||
| catch (KeyNotFoundException) { } | ||
| catch (ArgumentOutOfRangeException) { } | ||
|
|
||
| return null; | ||
| } | ||
|
|
||
| [Export("indexOfAccessibilityElement:")] | ||
| public nint GetIndexOfAccessibilityElement(NSObject element) | ||
| { | ||
| int indexOf = _childrenList.IndexOf((AutomationPeerWrapper)element); | ||
| return indexOf < 0 ? NSRange.NotFound : indexOf; | ||
| } | ||
|
|
||
| internal void UpdateChildren(AutomationPeer peer) | ||
| { | ||
| foreach (AutomationPeer child in peer.GetChildren()) | ||
| { | ||
| AutomationPeerWrapper? wrapper; | ||
| if (!_childrenMap.TryGetValue(child, out wrapper) && | ||
| (child.GetName().Length > 0 || child.IsKeyboardFocusable())) | ||
| { | ||
| _childrenList.Add(child); | ||
| _childrenMap.Add(child, new(this, child)); | ||
| } | ||
|
|
||
| wrapper?.UpdatePropertiesIfValid(); | ||
| wrapper?.UpdateTraits(); | ||
|
|
||
| UpdateChildren(child); | ||
| } | ||
| } | ||
|
There was a problem hiding this comment. I am not familiar with iOS accessibility, but shouldn't this backend map avalonia automation onto accessibilityContainer? In other words, it should have a tree of peer wrappers, implementing IUIAccessibilityContainer, on wrapping children only on request. Instead of processing every element in the app into a single dictionary, updating it on each accessibility request. Otherwise, it needs to be carefully tested for performance and memory leaks, and potentially that problem with TabControl I mentioned above. At the moment, cleanup is only happening on GetAccessibilityElementAt request for a specific element, and if iOS doesn't call this method, no element is cleared, holding it in the memory. By implementing IUIAccessibilityContainer on every node it would make it possible to properly handle accessibilityContainerType. There was a problem hiding this comment. The issue I encountered trying to implement what you're describing is that iOS then doesn't traverse the tree when using touch exploration. In other words, it ends up just settling on the root element because, like you are saying, it doesn't traverse deeper due to the OS not requesting that traversal in the first place. Unless I were to do some kind of implicit aggregation in the accessibilityContainer methods that does this traversal for the OS (which would again bring up the performance concerns), this is the only way I was able to get touch exploration to (mostly) work. All this said, I can definitely double check the cleanup code to make sure that we don't get the weird TabControl behavior you're describing. |
||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
cc @grokys, is it fine to switch to Effectively Visible? Or do other backends handle inherited visibility on their own?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Other backends handle this only in-so-far as they handle the hit testing in Avalonia code. This is in contrast to iOS which is the only target platform that performs the hit testing on the OS side. Therefore, I argue that this adaptation is necessary.