From ed431e95ee6036c2fae5b1e43d5ad990fa303222 Mon Sep 17 00:00:00 2001 From: Daniel Toyama Date: Tue, 7 May 2024 07:39:08 -0700 Subject: [PATCH] Introduce Accessibility Protobuf messages. These messages encode accessibility information coming from Android OS. Actual components that use them will come later. PiperOrigin-RevId: 631418985 --- android_env/proto/a11y/a11y.proto | 76 +++++++++++ .../a11y/android_accessibility_action.proto | 33 +++++ .../a11y/android_accessibility_forest.proto | 30 +++++ .../android_accessibility_node_info.proto | 123 ++++++++++++++++++ ...cessibility_node_info_clickable_span.proto | 50 +++++++ .../a11y/android_accessibility_tree.proto | 30 +++++ .../android_accessibility_window_info.proto | 85 ++++++++++++ android_env/proto/a11y/rect.proto | 31 +++++ 8 files changed, 458 insertions(+) create mode 100644 android_env/proto/a11y/a11y.proto create mode 100644 android_env/proto/a11y/android_accessibility_action.proto create mode 100644 android_env/proto/a11y/android_accessibility_forest.proto create mode 100644 android_env/proto/a11y/android_accessibility_node_info.proto create mode 100644 android_env/proto/a11y/android_accessibility_node_info_clickable_span.proto create mode 100644 android_env/proto/a11y/android_accessibility_tree.proto create mode 100644 android_env/proto/a11y/android_accessibility_window_info.proto create mode 100644 android_env/proto/a11y/rect.proto diff --git a/android_env/proto/a11y/a11y.proto b/android_env/proto/a11y/a11y.proto new file mode 100644 index 0000000..5c37f35 --- /dev/null +++ b/android_env/proto/a11y/a11y.proto @@ -0,0 +1,76 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package android_env; + +import "third_party/py/android_env/proto/a11y/android_accessibility_forest.proto"; + +option java_multiple_files = true; +option use_java_stubby_library = true; +option java_package = "com.google.androidenv.accessibilityforwarder"; + +// A service to send Accessibility information to a remote server. +// +// The client is assumed to be running inside an Android device (e.g. emulator +// or real device) while the server is assumed to be running outside (e.g. in a +// Python process). +service A11yService { + // Sends a forest of Accessibility trees to a server. + rpc SendForest(AndroidAccessibilityForest) returns (ForestResponse) {} + // Sends an a11y event to a server. + rpc SendEvent(EventRequest) returns (EventResponse) {} + + // Long-lived bidirection communication between the client and the server. + rpc Bidi(stream ClientToServer) returns (stream ServerToClient) {} +} + +// TODO(b/334952387): Remove `ForestResponse`, `EventRequest` and +// `EventResponse` once bidi communication is in-place. +message ForestResponse { + // The error if anything. + string error = 1; +} + +// An Accessibility event. +message EventRequest { + // A single event as a dictionary. + map event = 1; +} + +message EventResponse { + // The error if anything. + string error = 1; +} + +// The message sent from the Android device to the server running outside of the +// device. +message ClientToServer { + oneof payload { + EventRequest event = 1; + AndroidAccessibilityForest forest = 2; + } +} + +// The message sent from the server running outside of the device to the Android +// device. +message ServerToClient { + // A request to obtain the Accessibility forest. + message GetA11yForest {} + + oneof payload { + GetA11yForest get_forest = 1; + } +} diff --git a/android_env/proto/a11y/android_accessibility_action.proto b/android_env/proto/a11y/android_accessibility_action.proto new file mode 100644 index 0000000..1658c84 --- /dev/null +++ b/android_env/proto/a11y/android_accessibility_action.proto @@ -0,0 +1,33 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package android_env; + +option java_multiple_files = true; +option java_package = "com.google.androidenv.accessibilityforwarder"; +option java_api_version = 2; + +// An Android Accessibility Action. +// Next index: 3 +message AndroidAccessibilityAction { + // Required ID that uniquely identifies the action for this node. + // Can be one of the standard action IDs listed in the documentation. + // https://developer.android.com/reference/android/view/accessibility/AccessibilityNodeInfo.AccessibilityAction + int32 id = 1; + + // Optional label describing what the action is. + string label = 2; +} diff --git a/android_env/proto/a11y/android_accessibility_forest.proto b/android_env/proto/a11y/android_accessibility_forest.proto new file mode 100644 index 0000000..bc4ab95 --- /dev/null +++ b/android_env/proto/a11y/android_accessibility_forest.proto @@ -0,0 +1,30 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package android_env; + +import "third_party/py/android_env/proto/a11y/android_accessibility_window_info.proto"; + +option java_multiple_files = true; +option java_package = "com.google.androidenv.accessibilityforwarder"; +option java_api_version = 2; + +// A forest of Android accessibility trees. Each tree belongs to a single +// window. Next index: 2 +message AndroidAccessibilityForest { + // All of the windows present on screen. + repeated AndroidAccessibilityWindowInfo windows = 1; +} diff --git a/android_env/proto/a11y/android_accessibility_node_info.proto b/android_env/proto/a11y/android_accessibility_node_info.proto new file mode 100644 index 0000000..6ed0cd7 --- /dev/null +++ b/android_env/proto/a11y/android_accessibility_node_info.proto @@ -0,0 +1,123 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package android_env; + +import "third_party/py/android_env/proto/a11y/android_accessibility_action.proto"; +import "third_party/py/android_env/proto/a11y/android_accessibility_node_info_clickable_span.proto"; +import "third_party/py/android_env/proto/a11y/rect.proto"; + +option java_multiple_files = true; +option java_package = "com.google.androidenv.accessibilityforwarder"; +option java_api_version = 2; + +// An Android AccessibilityNodeInfo. +// Next index: 32 +message AndroidAccessibilityNodeInfo { + // Unique monotonically-increasing ID. + int32 unique_id = 1; + + // The bounds of this node within the device's screen. + ProtoRect bounds_in_screen = 2; + + // The name of the View class that created this node. + string class_name = 3; + + // The content description of the node. + string content_description = 4; + + // The hint text of the node. + string hint_text = 5; + + // The name of the package this node comes from. + string package_name = 6; + + // The text of this node. + string text = 7; + + // The start index of the text selection. + int64 text_selection_start = 8; + + // The end index of the text selection. + int64 text_selection_end = 9; + + // The view ID resource name of the node. + string view_id_resource_name = 10; + + // The ID of the window this node belongs to. + int32 window_id = 11; + + // If true, this node can be checked. + bool is_checkable = 12; + + // If true, this node is currently checked. + bool is_checked = 13; + + // If true, this node (probably) responds to being clicked. + bool is_clickable = 14; + + // If true, this node's text can be edited by the user. + bool is_editable = 15; + + // If true, this node is enabled (e.g., if it is a button). + bool is_enabled = 16; + + // If true, this node can be focused (e.g., a text input). + bool is_focusable = 17; + + // If true, this node is currently focused. + bool is_focused = 18; + + // If true, this node (probably) responds to being long pressed. + bool is_long_clickable = 19; + + // If true, this node is a password input. + bool is_password = 20; + + // If true, this node can be scrolled. + bool is_scrollable = 21; + + // If true, this node is currently selected. + bool is_selected = 22; + + // If true, this node is (probably) visible to the user. + bool is_visible_to_user = 23; + + // List of actions that can be performed on this node. + repeated AndroidAccessibilityAction actions = 24; + + // Ordered list of child IDs (i.e., unique_id). + repeated int32 child_ids = 25 [packed = true]; + + // List of clickable spans present in the node's text or content description. + repeated AndroidAccessibilityNodeInfoClickableSpan clickable_spans = 26; + + // The depth of this node in the accessibility tree. + int32 depth = 27; + + // Unique ID of the node that this node is declaring itself to be labeled by. + int32 labeled_by_id = 28; + + // Unique ID of the node that this is node is declaring itself to be a label + // for. + int32 label_for_id = 29; + + // The drawing order for the node. + int32 drawing_order = 30; + + // The tooltip text of the node. + string tooltip_text = 31; +} diff --git a/android_env/proto/a11y/android_accessibility_node_info_clickable_span.proto b/android_env/proto/a11y/android_accessibility_node_info_clickable_span.proto new file mode 100644 index 0000000..c60cade --- /dev/null +++ b/android_env/proto/a11y/android_accessibility_node_info_clickable_span.proto @@ -0,0 +1,50 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package android_env; + +option java_multiple_files = true; +option java_package = "com.google.androidenv.accessibilityforwarder"; +option java_api_version = 2; + +// A single clickable span found in the accessibility node's text. +// Next index: 6 +message AndroidAccessibilityNodeInfoClickableSpan { + // The source of the span (so the client can find the correct spannable string + // in the node). + // Next index: 3 + enum SpanSource { + UNKNOWN_TYPE = 0; // Catch all type for forward compatibility. + TEXT = 1; // The span is from node#getText + CONTENT_DESCRIPTION = 2; // The span is from node#getContentDescription. + } + + // The text of the span (a substring of the spannable string). + string text = 1; + + // The URL attached to the span if specified. + string url = 2; + + // The source of the span. + SpanSource source = 3; + + // The index of the first character of the span in the spannable string. + // The end of the span would be a sum of span_start and text.length(). + int32 start = 4; + + // The unique_id from the corresponding AndroidAccessibilityNodeInfo. + int32 node_id = 5; +} diff --git a/android_env/proto/a11y/android_accessibility_tree.proto b/android_env/proto/a11y/android_accessibility_tree.proto new file mode 100644 index 0000000..be77615 --- /dev/null +++ b/android_env/proto/a11y/android_accessibility_tree.proto @@ -0,0 +1,30 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package android_env; + +import "third_party/py/android_env/proto/a11y/android_accessibility_node_info.proto"; + +option java_multiple_files = true; +option java_package = "com.google.androidenv.accessibilityforwarder"; +option java_api_version = 2; + +// A tree (actually a graph) of Android accessibility nodes. +// Next index: 3 +message AndroidAccessibilityTree { + // All of the nodes in the graph. The root node is the node whose ID is 0. + repeated AndroidAccessibilityNodeInfo nodes = 1; +} diff --git a/android_env/proto/a11y/android_accessibility_window_info.proto b/android_env/proto/a11y/android_accessibility_window_info.proto new file mode 100644 index 0000000..a9f0935 --- /dev/null +++ b/android_env/proto/a11y/android_accessibility_window_info.proto @@ -0,0 +1,85 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package android_env; + +import "third_party/py/android_env/proto/a11y/android_accessibility_tree.proto"; +import "third_party/py/android_env/proto/a11y/rect.proto"; + +option java_multiple_files = true; +option java_package = "com.google.androidenv.accessibilityforwarder"; +option java_api_version = 2; + +// An Android AccessibilityWindowInfo. +// Next index: 12 +message AndroidAccessibilityWindowInfo { + // Type of the window. + // Next index: 6 + enum WindowType { + // The window type is an unknown value. + UNKNOWN_TYPE = 0; + + // A standard application window. + TYPE_APPLICATION = 1; + + // An IME window (e.g. GBoard). + TYPE_INPUT_METHOD = 2; + + // A system window (e.g., a notification). + TYPE_SYSTEM = 3; + + // An accessibility overlay. + TYPE_ACCESSIBILITY_OVERLAY = 4; + + // A system window used to divide the screen in split-screen mode. This type + // of window is present only in split-screen mode. + TYPE_SPLIT_SCREEN_DIVIDER = 5; + } + + // Bounds of this window in the device's screen. + ProtoRect bounds_in_screen = 1; + + // A unique ID identifying the display in which this window is shown. + int32 display_id = 2; + + // Unique ID as defined by the Android platform. + int32 id = 3; + + // Z-index of the window. Windows with a greater z-index appear in front of + // those with a lesser z-index. + int32 layer = 4; + + // The title of the window, if set. + string title = 5; + + // The type of the window. + WindowType window_type = 6; + + // If true, the window is currently accessibility-focused. + bool is_accessibility_focused = 7; + + // If true, the window is currently active. + bool is_active = 8; + + // If true, the window is currently focused. + bool is_focused = 9; + + // If true, the window is in Picture in Picture mode. + bool is_in_picture_in_picture_mode = 10; + + // The associated accessibility tree for this window. + AndroidAccessibilityTree tree = 11; +} diff --git a/android_env/proto/a11y/rect.proto b/android_env/proto/a11y/rect.proto new file mode 100644 index 0000000..6240cf1 --- /dev/null +++ b/android_env/proto/a11y/rect.proto @@ -0,0 +1,31 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package android_env; + +option java_multiple_files = true; +option java_package = "com.google.androidenv.accessibilityforwarder"; +option java_api_version = 2; + +// Proto representation of Android Rect. +// https://developer.android.com/reference/android/graphics/Rect +// Next index: 5 +message ProtoRect { + int32 left = 1; + int32 top = 2; + int32 right = 3; + int32 bottom = 4; +}