diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarder.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarder.kt new file mode 100644 index 0000000..6e9bf82 --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarder.kt @@ -0,0 +1,280 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.androidenv.accessibilityforwarder + +import android.accessibilityservice.AccessibilityService +import android.util.Log +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import com.google.androidenv.accessibilityforwarder.A11yServiceGrpcKt.A11yServiceCoroutineStub +import io.grpc.ManagedChannel +import io.grpc.ManagedChannelBuilder +import io.grpc.ProxyDetector +import io.grpc.StatusException +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout + +/** + * An Android service that listens to accessibility events and forwards them via gRPC. + * + * This service also logs the accessibility tree if [LogFlags.logAccessibilityTree] is set and if + * [LogFlags.grpcPort] is positive. + * + * Please see + * https://developer.android.com/reference/android/view/accessibility/AccessibilityEvent#getEventType() + * for a comprehensive list of events emitted by Android. + */ +class AccessibilityForwarder( + private val channelFactory: (host: String, port: Int) -> ManagedChannel = { host, port -> + ManagedChannelBuilder.forAddress(host, port) + .proxyDetector(ProxyDetector { _ -> null }) + .usePlaintext() + .build() + } +) : AccessibilityService() { + + init { + // Spawn long-running thread for periodically logging the tree. + Thread( + Runnable { + while (LogFlags.a11yTreePeriodMs > 0) { + try { + val windows = this.windows + logAccessibilityTree(windows) + } catch (e: ConcurrentModificationException) { + continue + } + + Thread.sleep(/* millis= */ LogFlags.a11yTreePeriodMs) + } + } + ) + .start() + } + + // grpcStub has a backing property that can be reset to null. + private var _grpcStub: A11yServiceCoroutineStub? = null + val grpcStub: A11yServiceCoroutineStub + get() { + if (_grpcStub == null) { + Log.i(TAG, "Building channel on ${LogFlags.grpcHost}:${LogFlags.grpcPort}.") + _grpcStub = A11yServiceCoroutineStub(channelFactory(LogFlags.grpcHost, LogFlags.grpcPort)) + } + return _grpcStub!! + } + + private fun resetGrpcStub() { + _grpcStub = null + } + + override fun onInterrupt() { + LogFlags.a11yTreePeriodMs = 0 // Turn off periodic tree forwarding. + } + + override fun onAccessibilityEvent(event: AccessibilityEvent?) { + if (event == null) { + Log.i(TAG, "`event` is null.") + return + } + + logExtrasForEvent(event) + val eventType = event.eventType + val eventTypeStr: String = AccessibilityEvent.eventTypeToString(eventType) + if (eventTypeStr.isNotEmpty()) { + Log.i(TAG, eventTypeStr) + } + } + + private fun logAccessibilityTree(windows: List) { + if (!LogFlags.logAccessibilityTree) { + Log.i(TAG, "Not logging accessibility tree") + return + } + + // Check gRPC port before actually building the forest. + if (LogFlags.grpcPort <= 0) { + Log.w(TAG, "Can't log accessibility tree because gRPC port has not been set.") + return + } + + val forest = creator.buildForest(windows) + try { + val grpcTimeoutMillis = 1000L + val response: ForestResponse = + with(grpcStub) { + Log.i(TAG, "sending (blocking) gRPC request for tree.") + runBlocking { withTimeout(grpcTimeoutMillis) { sendForest(forest) } } + } + if (response.error.isNotEmpty()) { + Log.w(TAG, "gRPC response.error: ${response.error}") + } else { + Log.i(TAG, "gRPC request for tree succeeded.") + } + } catch (e: StatusException) { + Log.w(TAG, "gRPC StatusException; are you sure networking is turned on?") + Log.i(TAG, "extra: exception ['$e']") + resetGrpcStub() + } catch (e: TimeoutCancellationException) { + Log.w(TAG, "gRPC TimeoutCancellationException; are you sure networking is turned on?") + Log.i(TAG, "extra: exception ['$e']") + resetGrpcStub() + } + } + + /** Logs extras for all event types. */ + private fun logExtrasForEvent(event: AccessibilityEvent) { + + val events: MutableMap = mutableMapOf() + + val sourceDescription = event.source?.contentDescription() + if (!sourceDescription.isNullOrEmpty()) { + events.put("source_content_description", sourceDescription) + } + + // Output the event text. + val eventText = event.text.joinToString(", ") + if (eventText.isNotEmpty()) { + events.put("event_text", eventText) + } + + // Output the source text. + val sourceText = event.source?.text?.toString() + if (!sourceText.isNullOrEmpty()) { + events.put("source_text", sourceText) + } + + val eventTypeStr: String = AccessibilityEvent.eventTypeToString(event.eventType) + if (eventTypeStr.isNotEmpty()) { + events.put("event_type", eventTypeStr) + } + + val className = event.source?.className?.toString() + if (!className.isNullOrEmpty()) { + events.put("source_class_name", className) + } + + val packageName = event.packageName?.toString() + if (!packageName.isNullOrEmpty()) { + events.put("event_package_name", packageName) + } + + // Text editing properties. + val beforeText = event.beforeText?.toString() + if (!beforeText.isNullOrEmpty()) { + events.put("before_text", beforeText) + } + + val fromIndex = event.fromIndex + if (fromIndex != -1) { + events.put("from_index", fromIndex.toString()) + } + + val toIndex = event.toIndex + if (toIndex != -1) { + events.put("to_index", toIndex.toString()) + } + + val addedCount = event.addedCount + if (addedCount != -1) { + events.put("added_count", addedCount.toString()) + } + + val removedCount = event.removedCount + if (removedCount != -1) { + events.put("removed_count", removedCount.toString()) + } + + // Text traversal properties + val movementGranularity = event.movementGranularity + if (movementGranularity != 0) { + events.put("movement_granularity", movementGranularity.toString()) + } + + val action = event.action + if (action != 0) { + events.put("action", action.toString()) + } + + // Scrolling properties. + if (eventTypeStr == "TYPE_VIEW_SCROLLED") { + events.put("scroll_delta_x", event.scrollDeltaX.toString()) + events.put("scroll_delta_y", event.scrollDeltaY.toString()) + } + + // Report viewID so we know exactly where the event came from. + val viewId = event.source?.viewIdResourceName?.toString() + if (!viewId.isNullOrEmpty()) { + events.put("view_id", viewId) + } + + // Format [events] as a Python dict. + if (events.isNotEmpty()) { + events.put("event_timestamp_ms", event.eventTime.toString(10)) + // Check if we want to use gRPC. + if (LogFlags.grpcPort > 0) { + try { + val grpcTimeoutMillis = 1000L + val request = eventRequest { this.event.putAll(events) } + val response: EventResponse = + with(grpcStub) { + Log.i(TAG, "sending (blocking) gRPC request for event.") + runBlocking { withTimeout(grpcTimeoutMillis) { sendEvent(request) } } + } + if (response.error.isNotEmpty()) { + Log.w(TAG, "gRPC response.error: ${response.error}") + } else { + Log.i(TAG, "gRPC request for event succeeded.") + } + } catch (e: StatusException) { + Log.w(TAG, "gRPC StatusException; are you sure networking is turned on?") + Log.i(TAG, "extra: exception ['$e']") + resetGrpcStub() + } catch (e: TimeoutCancellationException) { + Log.w(TAG, "gRPC TimeoutCancellationException; are you sure networking is turned on?") + Log.i(TAG, "extra: exception ['$e']") + resetGrpcStub() + } + } else { + Log.w(TAG, "Can't log accessibility event because gRPC port has not been set.") + } + } + } + + /** Recursively climbs the accessibility tree until the root, collecting descriptions. */ + private fun AccessibilityNodeInfo?.contentDescription(): String { + if (this == null) { + return "" + } + + val descriptions = mutableListOf() + var current: AccessibilityNodeInfo? = this + while (current != null) { + val description = current.contentDescription + if (description != null) { + descriptions.add(description.toString()) + } + + current = current.parent + } + return descriptions.joinToString(", ") + } + + companion object { + private const val TAG = "AndroidRLTask" + private val creator = AccessibilityTreeCreator() + } +} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarderTest.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarderTest.kt new file mode 100644 index 0000000..6698ce6 --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarderTest.kt @@ -0,0 +1,516 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.androidenv.accessibilityforwarder + +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import com.google.common.truth.Truth.assertThat +import com.google.thirdparty.robolectric.testparameterinjector.RobolectricTestParameterInjector +import io.grpc.Status +import io.grpc.StatusException +import io.grpc.inprocess.InProcessChannelBuilder +import io.grpc.inprocess.InProcessServerBuilder +import io.grpc.testing.GrpcCleanupRule +import org.junit.Assert.assertFalse +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Shadows.shadowOf + +@RunWith(RobolectricTestParameterInjector::class) +class AccessibilityForwarderTest { + + @get:Rule(order = 1) val cleanupRule = GrpcCleanupRule() + + class FakeAccessibilityService : A11yServiceGrpcKt.A11yServiceCoroutineImplBase() { + var sendForestChecker: (AndroidAccessibilityForest) -> String = { _ -> "" } + var sendEventChecker: (EventRequest) -> String = { _ -> "" } + + override suspend fun sendForest(request: AndroidAccessibilityForest) = forestResponse { + error = sendForestChecker(request) + } + + override suspend fun sendEvent(request: EventRequest) = eventResponse { + error = sendEventChecker(request) + } + } + + protected lateinit var forwarder: AccessibilityForwarder + protected val fakeA11yService = FakeAccessibilityService() + protected val channel by lazy { + val serverName: String = InProcessServerBuilder.generateName() + cleanupRule.register( + InProcessServerBuilder.forName(serverName) + .directExecutor() + .addService(fakeA11yService) + .build() + .start() + ) + cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()) + } + + /** Initializes [forwarder] and [LogFlags] from the given args. */ + fun createForwarder( + logAccessibilityTree: Boolean = false, + a11yTreePeriodMs: Long = 0, + grpcHost: String = "10.0.2.2", + grpcPort: Int = 0, + a11yWindows: MutableList? = null, + ) { + LogFlags.logAccessibilityTree = logAccessibilityTree + LogFlags.a11yTreePeriodMs = a11yTreePeriodMs + LogFlags.grpcHost = grpcHost + LogFlags.grpcPort = grpcPort + forwarder = AccessibilityForwarder({ _, _ -> channel }) + if (a11yWindows == null) { + shadowOf(forwarder).setWindows(mutableListOf(AccessibilityWindowInfo.obtain())) + } else { + shadowOf(forwarder).setWindows(a11yWindows) + } + } + + @Test + fun onInterrupt_doesNotCrash() { + // Arrange. + createForwarder(logAccessibilityTree = false) + fakeA11yService.sendEventChecker = { _: EventRequest -> + assertFalse(true) // This should not be called. + "" // This should be unreachable + } + + // Act. + forwarder.onInterrupt() + + // Assert. + // See `sendEventChecker` above. + } + + @Test + fun onAccessibilityEvent_nullEventShouldBeIgnored() { + // Arrange. + createForwarder(logAccessibilityTree = false) + fakeA11yService.sendEventChecker = { _: EventRequest -> + assertFalse(true) // This should not be called. + "" // This should be unreachable + } + + // Act. + forwarder.onAccessibilityEvent(null) + + // Assert. + // See `sendEventChecker` above. + } + + @Test + fun onAccessibilityEvent_knownEventWithNoInformationShouldNotBeEmitted() { + // Arrange. + createForwarder(logAccessibilityTree = false) + var nodeInfo = AccessibilityNodeInfo() + nodeInfo.setContentDescription("") + var event = AccessibilityEvent() + shadowOf(event).setSourceNode(nodeInfo) + fakeA11yService.sendEventChecker = { _: EventRequest -> + assertFalse(true) // This should not be called. + "" // This should be unreachable + } + + // Act. + forwarder.onAccessibilityEvent(event) + + // Assert. + // See `sendEventChecker` above. + } + + @Test + fun onAccessibilityEvent_typeViewClicked_sendEventViaGrpc() { + // Arrange. + createForwarder(logAccessibilityTree = false, grpcPort = 1234) + forwarder = AccessibilityForwarder({ _, _ -> channel }) + var nodeInfo = AccessibilityNodeInfo() + nodeInfo.setContentDescription("My Content Description") + nodeInfo.setText("My Source Text") + nodeInfo.setClassName("AwesomeClass") + var event = AccessibilityEvent() + event.setEventTime(1357924680) + event.setEventType(AccessibilityEvent.TYPE_VIEW_CLICKED) + event.getText().add("Some text!") + event.setPackageName("some.loooong.package.name") + shadowOf(event).setSourceNode(nodeInfo) + fakeA11yService.sendEventChecker = { request: EventRequest -> + // Check that all fields are consistent with how they were set above. + assertThat(request.eventMap.get("event_type")).isEqualTo("TYPE_VIEW_CLICKED") + assertThat(request.eventMap.get("event_package_name")).isEqualTo("some.loooong.package.name") + assertThat(request.eventMap.get("source_content_description")) + .isEqualTo("My Content Description") + assertThat(request.eventMap.get("source_text")).isEqualTo("My Source Text") + assertThat(request.eventMap.get("source_class_name")).isEqualTo("AwesomeClass") + assertThat(request.eventMap.get("event_text")).isEqualTo("Some text!") + assertThat(request.eventMap.get("event_timestamp_ms")).isEqualTo("1357924680") + // No error message + "" + } + + // Act. + forwarder.onAccessibilityEvent(event) + + // Assert. + // See `sendEventChecker` above. + } + + @Test + fun onAccessibilityEvent_typeViewTextChanged_ensureAllFieldsForwarded() { + // Arrange. + createForwarder(logAccessibilityTree = false, grpcPort = 1234) + var nodeInfo = AccessibilityNodeInfo() + nodeInfo.setContentDescription("My Content Description") + nodeInfo.setText("My Source Text") + nodeInfo.setClassName("AwesomeClass") + var event = AccessibilityEvent() + event.setEventTime(1357924680) + event.setEventType(AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED) + event.getText().add("Some text!") + event.fromIndex = 7 + event.beforeText = "Old words" + event.addedCount = 12 + event.removedCount = 9 + event.setPackageName("some.loooong.package.name") + shadowOf(event).setSourceNode(nodeInfo) + fakeA11yService.sendEventChecker = { request: EventRequest -> + // Check that all fields are consistent with how they were set above. + assertThat(request.eventMap.get("event_type")).isEqualTo("TYPE_VIEW_TEXT_CHANGED") + assertThat(request.eventMap.get("event_package_name")).isEqualTo("some.loooong.package.name") + assertThat(request.eventMap.get("source_content_description")) + .isEqualTo("My Content Description") + assertThat(request.eventMap.get("source_text")).isEqualTo("My Source Text") + assertThat(request.eventMap.get("source_class_name")).isEqualTo("AwesomeClass") + assertThat(request.eventMap.get("event_text")).isEqualTo("Some text!") + assertThat(request.eventMap.get("event_timestamp_ms")).isEqualTo("1357924680") + assertThat(request.eventMap.get("from_index")).isEqualTo("7") + assertThat(request.eventMap.get("before_text")).isEqualTo("Old words") + assertThat(request.eventMap.get("added_count")).isEqualTo("12") + assertThat(request.eventMap.get("removed_count")).isEqualTo("9") + assertFalse(request.eventMap.containsKey("to_index")) + assertFalse(request.eventMap.containsKey("view_id")) + assertFalse(request.eventMap.containsKey("action")) + assertFalse(request.eventMap.containsKey("movement_granularity")) + assertFalse(request.eventMap.containsKey("scroll_delta_x")) + assertFalse(request.eventMap.containsKey("scroll_delta_y")) + // No error message + "" + } + + // Act. + forwarder.onAccessibilityEvent(event) + + // Assert. + // See `sendEventChecker` above. + } + + @Test + fun onAccessibilityEvent_typeViewScrolled_ensureAllFieldsForwarded() { + // Arrange. + createForwarder(logAccessibilityTree = false, grpcPort = 1234) + var nodeInfo = AccessibilityNodeInfo() + nodeInfo.setContentDescription("My Content Description") + nodeInfo.setText("My Source Text") + nodeInfo.setClassName("AwesomeClass") + var event = AccessibilityEvent() + event.setEventTime(1357924680) + event.setEventType(AccessibilityEvent.TYPE_VIEW_SCROLLED) + event.getText().add("Some text!") + event.scrollDeltaX = 13 + event.scrollDeltaY = 27 + event.setPackageName("some.loooong.package.name") + shadowOf(event).setSourceNode(nodeInfo) + fakeA11yService.sendEventChecker = { request: EventRequest -> + // Check that all fields are consistent with how they were set above. + assertThat(request.eventMap.get("event_type")).isEqualTo("TYPE_VIEW_SCROLLED") + assertThat(request.eventMap.get("event_package_name")).isEqualTo("some.loooong.package.name") + assertThat(request.eventMap.get("source_content_description")) + .isEqualTo("My Content Description") + assertThat(request.eventMap.get("source_text")).isEqualTo("My Source Text") + assertThat(request.eventMap.get("source_class_name")).isEqualTo("AwesomeClass") + assertThat(request.eventMap.get("event_text")).isEqualTo("Some text!") + assertThat(request.eventMap.get("event_timestamp_ms")).isEqualTo("1357924680") + assertThat(request.eventMap.get("scroll_delta_x")).isEqualTo("13") + assertThat(request.eventMap.get("scroll_delta_y")).isEqualTo("27") + assertFalse(request.eventMap.containsKey("from_index")) + assertFalse(request.eventMap.containsKey("to_index")) + assertFalse(request.eventMap.containsKey("before_text")) + assertFalse(request.eventMap.containsKey("added_count")) + assertFalse(request.eventMap.containsKey("removed_count")) + // No error message + "" + } + + // Act. + forwarder.onAccessibilityEvent(event) + + // Assert. + // See `sendEventChecker` above. + } + + @Test + fun onAccessibilityEvent_typeViewTextTraversedAtMovementGranularity_ensureAllFieldsForwarded() { + // Arrange. + createForwarder(logAccessibilityTree = false, grpcPort = 1234) + var nodeInfo = AccessibilityNodeInfo() + nodeInfo.setContentDescription("My Content Description") + nodeInfo.setText("My Source Text") + nodeInfo.setClassName("AwesomeClass") + nodeInfo.viewIdResourceName = "this.big.old.view.id" + var event = AccessibilityEvent() + event.setEventTime(1357924680) + event.setEventType(AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY) + event.getText().add("Some text!") + event.setPackageName("some.loooong.package.name") + event.movementGranularity = 5 + event.fromIndex = 6 + event.toIndex = 8 + event.action = 23 + shadowOf(event).setSourceNode(nodeInfo) + fakeA11yService.sendEventChecker = { request: EventRequest -> + // Check that all fields are consistent with how they were set above. + assertThat(request.eventMap.get("event_type")) + .isEqualTo("TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY") + assertThat(request.eventMap.get("event_package_name")).isEqualTo("some.loooong.package.name") + assertThat(request.eventMap.get("source_content_description")) + .isEqualTo("My Content Description") + assertThat(request.eventMap.get("source_text")).isEqualTo("My Source Text") + assertThat(request.eventMap.get("source_class_name")).isEqualTo("AwesomeClass") + assertThat(request.eventMap.get("event_text")).isEqualTo("Some text!") + assertThat(request.eventMap.get("event_timestamp_ms")).isEqualTo("1357924680") + assertThat(request.eventMap.get("movement_granularity")).isEqualTo("5") + assertThat(request.eventMap.get("from_index")).isEqualTo("6") + assertThat(request.eventMap.get("to_index")).isEqualTo("8") + assertThat(request.eventMap.get("view_id")).isEqualTo("this.big.old.view.id") + assertThat(request.eventMap.get("action")).isEqualTo("23") + // No error message + "" + } + + // Act. + forwarder.onAccessibilityEvent(event) + + // Assert. + // See `sendEventChecker` above. + } + + @Test + fun onAccessibilityEvent_sendingevent_grpcTimeout() { + // Arrange. + createForwarder( + logAccessibilityTree = false, + a11yTreePeriodMs = 0, + grpcHost = "amazing.host", + grpcPort = 4321, + ) + var nodeInfo = AccessibilityNodeInfo() + nodeInfo.setContentDescription("My Content Description") + nodeInfo.setText("My Source Text") + nodeInfo.setClassName("AwesomeClass") + var event = AccessibilityEvent() + event.setEventTime(1357924680) + event.setEventType(AccessibilityEvent.TYPE_VIEW_CLICKED) + event.getText().add("Some text!") + event.setPackageName("some.loooong.package.name") + shadowOf(event).setSourceNode(nodeInfo) + fakeA11yService.sendEventChecker = { _ -> + // Delay the request to prompt a timeout + Thread.sleep(1500L) + "" // Return no error. + } + + // Act. + forwarder.onAccessibilityEvent(event) + + // Run a second request to ensure that the channel gets rebuilt. + fakeA11yService.sendEventChecker = { _ -> "" } + forwarder.onAccessibilityEvent(event) + + // Assert. + // See `sendEventChecker` above. + } + + @Test + fun onAccessibilityEvent_sendingevent_grpcStatusException() { + // Arrange. + createForwarder(logAccessibilityTree = false, grpcHost = "amazing.host", grpcPort = 4321) + var nodeInfo = AccessibilityNodeInfo() + nodeInfo.setContentDescription("My Content Description") + nodeInfo.setText("My Source Text") + nodeInfo.setClassName("AwesomeClass") + var event = AccessibilityEvent() + event.setEventTime(1357924680) + event.setEventType(AccessibilityEvent.TYPE_VIEW_CLICKED) + event.getText().add("Some text!") + event.setPackageName("some.loooong.package.name") + shadowOf(event).setSourceNode(nodeInfo) + fakeA11yService.sendEventChecker = { _ -> throw StatusException(Status.UNAVAILABLE) } + + // Act. + forwarder.onAccessibilityEvent(event) + + // Run a second request to ensure that the channel gets rebuilt. + fakeA11yService.sendEventChecker = { _ -> "" } + forwarder.onAccessibilityEvent(event) + + // Assert. + // See `sendEventChecker` above. + } + + @Test + fun logAccessibilityTreeFalse_doesNotLogAccessibilityTree() { + // Arrange. + createForwarder(logAccessibilityTree = false, a11yTreePeriodMs = 10, grpcPort = 13579) + fakeA11yService.sendForestChecker = { _: AndroidAccessibilityForest -> + assertFalse(true) // This should not be called. + "" // This should be unreachable + } + + // Act. + Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. + + // Assert. + // See `sendForestChecker` above. + } + + @Test + fun grpcPortZero_doesNotSendTree() { + // Arrange. + createForwarder(logAccessibilityTree = true, a11yTreePeriodMs = 10, grpcPort = 0) + fakeA11yService.sendForestChecker = { _: AndroidAccessibilityForest -> + assertFalse(true) // This should not be called. + "" // This should be unreachable + } + + // Act. + Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. + + // Assert. + // See `sendForestChecker` above. + } + + @Test + fun grpcPortPositive_shouldSendTreeViaGrpc() { + // Arrange. + val window = AccessibilityWindowInfo() + shadowOf(window).setType(AccessibilityWindowInfo.TYPE_SYSTEM) + createForwarder( + logAccessibilityTree = true, + a11yTreePeriodMs = 10, + grpcPort = 1234, + a11yWindows = mutableListOf(window), + ) + fakeA11yService.sendForestChecker = { request: AndroidAccessibilityForest -> + // Check that we get only a single window. + assertThat(request.windowsList.size).isEqualTo(1) + // And that its type is what we set above. + assertThat(request.windowsList[0].windowType) + .isEqualTo(AndroidAccessibilityWindowInfo.WindowType.TYPE_SYSTEM) + // The error message + "Something went wrong!" + } + + // Act. + Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. + + // Assert. + // See `sendForestChecker` above. + } + + @Test + fun grpcPortPositiveAndHost_shouldSendTreeViaGrpc() { + // Arrange. + fakeA11yService.sendForestChecker = { request: AndroidAccessibilityForest -> + // Check that we get only a single window. + assertThat(request.windowsList.size).isEqualTo(1) + // And that its type is what we set above. + assertThat(request.windowsList[0].windowType) + .isEqualTo(AndroidAccessibilityWindowInfo.WindowType.TYPE_ACCESSIBILITY_OVERLAY) + "" // Return no error. + } + val window = AccessibilityWindowInfo() + shadowOf(window).setType(AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) + createForwarder( + logAccessibilityTree = true, + a11yTreePeriodMs = 500, + grpcHost = "amazing.host", + grpcPort = 4321, + a11yWindows = mutableListOf(window), + ) + + // Act. + Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. + + // Assert. + // See `sendForestChecker` above. + } + + @Test + fun sendingForest_grpcTimeout() { + // Arrange. + fakeA11yService.sendForestChecker = { _ -> + // Delay the request to prompt a timeout + Thread.sleep(1500L) + "" // Return no error. + } + val window = AccessibilityWindowInfo() + shadowOf(window).setType(AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) + createForwarder( + logAccessibilityTree = true, + a11yTreePeriodMs = 10, + grpcHost = "amazing.host", + grpcPort = 4321, + a11yWindows = mutableListOf(window), + ) + + // Act. + Thread.sleep(2000) // Sleep a bit to give time to trigger the tree logging function. + + // Run a second request to ensure that the channel gets rebuilt. + fakeA11yService.sendForestChecker = { _ -> "" } + Thread.sleep(2000) // Sleep a bit to give time to trigger the tree logging function. + + // Assert. + // See `sendForestChecker` above. + } + + @Test + fun sendingForest_grpcStatusException() { + // Arrange. + val window = AccessibilityWindowInfo() + shadowOf(window).setType(AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) + createForwarder( + logAccessibilityTree = true, + a11yTreePeriodMs = 10, + grpcHost = "amazing.host", + grpcPort = 4321, + a11yWindows = mutableListOf(window), + ) + fakeA11yService.sendForestChecker = { _ -> throw StatusException(Status.UNAVAILABLE) } + + // Act. + Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. + + // Run a second request to ensure that the channel gets rebuilt. + fakeA11yService.sendForestChecker = { _ -> "" } + Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. + + // Assert. + // See `sendForestChecker` above. + } +} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreator.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreator.kt new file mode 100644 index 0000000..aeaee70 --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreator.kt @@ -0,0 +1,235 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.androidenv.accessibilityforwarder + +import android.graphics.Rect +import android.util.Log +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import com.google.androidenv.accessibilityforwarder.AndroidAccessibilityWindowInfo.WindowType +import java.util.concurrent.ConcurrentHashMap +import java.util.stream.Collectors +import kotlin.collections.mutableListOf +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking + +/** Helper methods for creating the android accessibility info extra. */ +class AccessibilityTreeCreator() { + + /** Creates an accessibility forest proto. */ + fun buildForest(windowInfos: List): AndroidAccessibilityForest { + val sourcesMap: ConcurrentHashMap = + ConcurrentHashMap() + val windows: List = + processWindowsAndBlock(windowInfos, sourcesMap) + return androidAccessibilityForest { this.windows += windows } + } + + private fun processWindowsAndBlock( + windowInfos: List, + sourcesMap: ConcurrentHashMap, + ): List { + val windows: List + runBlocking { windows = processWindows(windowInfos, sourcesMap) } + return windows + } + + private suspend fun processWindows( + windowInfos: List, + sourcesMap: ConcurrentHashMap, + ): List { + var windowInfoProtos = mutableListOf() + for (i in windowInfos.size - 1 downTo 0) { + val windowInfoProto = processWindow(windowInfos.get(i), sourcesMap) + windowInfoProto?.let { windowInfoProtos.add(windowInfoProto) } + } + return windowInfoProtos.toList() + } + + private suspend fun processWindow( + windowInfo: AccessibilityWindowInfo, + sources: ConcurrentHashMap, + ): AndroidAccessibilityWindowInfo? { + val bounds = Rect() + windowInfo.getBoundsInScreen(bounds) + val root: AccessibilityNodeInfo? = windowInfo.root + if (root == null) { + Log.i(TAG, "window root is null") + return androidAccessibilityWindowInfo { + this.tree = androidAccessibilityTree {} + this.isActive = windowInfo.isActive + this.id = windowInfo.id + this.layer = windowInfo.layer + this.isAccessibilityFocused = windowInfo.isAccessibilityFocused + this.isFocused = windowInfo.isFocused + this.boundsInScreen = convertToRectProto(bounds) + this.windowType = toWindowType(windowInfo.type) + } + } + val treeDeferred: Deferred + runBlocking { treeDeferred = async { processNodesInWindow(root, sources) } } + return androidAccessibilityWindowInfo { + this.tree = treeDeferred.await() + this.isActive = windowInfo.isActive + this.id = windowInfo.id + this.layer = windowInfo.layer + this.isAccessibilityFocused = windowInfo.isAccessibilityFocused + this.isFocused = windowInfo.isFocused + this.boundsInScreen = convertToRectProto(bounds) + this.windowType = toWindowType(windowInfo.type) + } + } + + private suspend fun processNodesInWindow( + root: AccessibilityNodeInfo, + sources: ConcurrentHashMap, + ): AndroidAccessibilityTree { + Log.d(TAG, "processNodesInWindow()") + val traversalQueue = ArrayDeque() + traversalQueue.add(ParentChildNodePair.builder().child(root).build()) + val uniqueIdsCache: UniqueIdsGenerator = UniqueIdsGenerator() + var currentDepth = 0 + val nodesDeferred = mutableListOf>() + val seenNodes: HashSet = HashSet() + seenNodes.add(root) + runBlocking { + while (!traversalQueue.isEmpty()) { + // Traverse the tree layer-by-layer. + // The first layer has only the root and depth 0. + // The second layer has all the root's children and depth 1. + for (nodesAtCurrentDepth in traversalQueue.size downTo 1) { + val nodePair: ParentChildNodePair = traversalQueue.removeFirst() + for (i in 0 until nodePair.child().childCount) { + val childNode: AccessibilityNodeInfo? = nodePair.child().getChild(i) + if (childNode != null && !seenNodes.contains(childNode)) { + traversalQueue.add( + ParentChildNodePair.builder().child(childNode).parent(nodePair.child()).build() + ) + seenNodes.add(childNode) + } + } + val thisDepth = currentDepth + var deferred = async { processNode(nodePair, sources, uniqueIdsCache, thisDepth) } + nodesDeferred.add(deferred) + } + currentDepth++ + } + } + return androidAccessibilityTree { this.nodes += nodesDeferred.awaitAll() } + } + + companion object { + private const val TAG = "AndroidRLTask" + } +} + +private fun processNode( + nodePair: ParentChildNodePair, + sourceBuilder: ConcurrentHashMap, + uniqueIdsCache: UniqueIdsGenerator, + nodeDepth: Int, +): AndroidAccessibilityNodeInfo { + val node: AccessibilityNodeInfo = nodePair.child() + val immutableNode: AndroidAccessibilityNodeInfo = + createAndroidAccessibilityNode( + node, + uniqueIdsCache.getUniqueId(node), + nodeDepth, + getChildUniqueIds(node, uniqueIdsCache), + ) + sourceBuilder.put(immutableNode, node) + return immutableNode +} + +private fun createAndroidAccessibilityNode( + node: AccessibilityNodeInfo, + nodeId: Int, + depth: Int, + childIds: List, +): AndroidAccessibilityNodeInfo { + val bounds = Rect() + node.getBoundsInScreen(bounds) + val actions = node.getActionList().stream().map(::createAction).collect(Collectors.toList()) + return androidAccessibilityNodeInfo { + this.actions += actions + this.boundsInScreen = convertToRectProto(bounds) + this.isCheckable = node.isCheckable + this.isChecked = node.isChecked + this.className = stringFromNullableCharSequence(node.getClassName()) + this.isClickable = node.isClickable + this.contentDescription = stringFromNullableCharSequence(node.getContentDescription()) + this.isEditable = node.isEditable + this.isEnabled = node.isEnabled + this.isFocusable = node.isFocusable + this.hintText = stringFromNullableCharSequence(node.getHintText()) + this.isLongClickable = node.isLongClickable + this.packageName = stringFromNullableCharSequence(node.getPackageName()) + this.isPassword = node.isPassword + this.isScrollable = node.isScrollable + this.isSelected = node.isSelected + this.text = stringFromNullableCharSequence(node.getText()) + this.textSelectionEnd = node.getTextSelectionEnd().toLong() + this.textSelectionStart = node.getTextSelectionStart().toLong() + this.viewIdResourceName = node.getViewIdResourceName() ?: "" + this.isVisibleToUser = node.isVisibleToUser + this.windowId = node.windowId + this.uniqueId = nodeId + this.childIds += childIds + this.drawingOrder = node.drawingOrder + this.tooltipText = stringFromNullableCharSequence(node.getTooltipText()) + this.depth = depth + } +} + +private fun createAction( + action: AccessibilityNodeInfo.AccessibilityAction +): AndroidAccessibilityAction = + AndroidAccessibilityAction.newBuilder() + .setId(action.id) + .setLabel(stringFromNullableCharSequence(action.label)) + .build() + +private fun getChildUniqueIds( + node: AccessibilityNodeInfo, + uniqueIdsCache: UniqueIdsGenerator, +): List { + val ids = mutableListOf() + for (childId in 0 until node.getChildCount()) { + val child: AccessibilityNodeInfo = node.getChild(childId) ?: continue + ids.add(uniqueIdsCache.getUniqueId(child)) + } + return ids.toList() +} + +fun stringFromNullableCharSequence(cs: CharSequence?): String = cs?.toString() ?: "" + +fun convertToRectProto(rect: Rect) = protoRect { + left = rect.left + top = rect.top + right = rect.right + bottom = rect.bottom +} + +private fun toWindowType(type: Int): WindowType = + when (type) { + AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY -> WindowType.TYPE_ACCESSIBILITY_OVERLAY + AccessibilityWindowInfo.TYPE_APPLICATION -> WindowType.TYPE_APPLICATION + AccessibilityWindowInfo.TYPE_INPUT_METHOD -> WindowType.TYPE_INPUT_METHOD + AccessibilityWindowInfo.TYPE_SYSTEM -> WindowType.TYPE_SYSTEM + AccessibilityWindowInfo.TYPE_SPLIT_SCREEN_DIVIDER -> WindowType.TYPE_SPLIT_SCREEN_DIVIDER + else -> WindowType.UNKNOWN_TYPE + } diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreatorTest.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreatorTest.kt new file mode 100644 index 0000000..b05d23a --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreatorTest.kt @@ -0,0 +1,85 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.androidenv.accessibilityforwarder + +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import kotlin.test.assertEquals +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf + +@RunWith(RobolectricTestRunner::class) +class AccessibilityTreeCreatorTest { + + @Test + fun buildForest_buildsAccessibilityForestCorrectly() { + val creator = AccessibilityTreeCreator() + + val forest = creator.buildForest(mutableListOf(createWindowInfo())) + + assertEquals(forest.windowsCount, 1) + assertEquals(forest.getWindows(0).tree.nodesCount, 3) + var rootNode: AndroidAccessibilityNodeInfo? = null + var checkableNode: AndroidAccessibilityNodeInfo? = null + val nodes = forest.getWindows(0).tree.nodesList + for (i in nodes.size - 1 downTo 0) { + if (nodes[i].text == "root node") { + rootNode = nodes[i] + } + if (nodes[i].isCheckable == true) { + checkableNode = nodes[i] + } + } + assertEquals(rootNode?.childIdsCount, 2) + assertEquals(checkableNode?.text, "Check box") + } + + @Test + fun buildForest_noRootInWindow_returnsEmptyTree() { + val creator = AccessibilityTreeCreator() + val windowInfo = AccessibilityWindowInfo.obtain() + shadowOf(windowInfo).setType(AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) + + val forest = creator.buildForest(mutableListOf(windowInfo)) + + assertEquals(0, forest.getWindows(0).tree.nodesList.size) + } + + private fun createAccessibilityNodeInfo(): AccessibilityNodeInfo { + val root = AccessibilityNodeInfo.obtain() + root.text = "root node" + root.isClickable = true + val accessibilityNodeInfo = AccessibilityNodeInfo.obtain() + accessibilityNodeInfo.viewIdResourceName = "test" + accessibilityNodeInfo.isClickable = true + accessibilityNodeInfo.isEditable = true + accessibilityNodeInfo.hintText = "Please enter your address" + shadowOf(root).addChild(accessibilityNodeInfo) + val anotherChildNode = AccessibilityNodeInfo.obtain() + anotherChildNode.isCheckable = true + anotherChildNode.text = "Check box" + shadowOf(root).addChild(anotherChildNode) + return root + } + + private fun createWindowInfo(): AccessibilityWindowInfo { + val windowInfo = AccessibilityWindowInfo.obtain() + shadowOf(windowInfo).setType(AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) + shadowOf(windowInfo).setRoot(createAccessibilityNodeInfo()) + return windowInfo + } +} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest.xml b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest.xml new file mode 100644 index 0000000..debf611 --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest.xml @@ -0,0 +1,44 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest_lite.xml b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest_lite.xml new file mode 100644 index 0000000..7bf2e85 --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest_lite.xml @@ -0,0 +1,21 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + + + + + + diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiver.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiver.kt new file mode 100644 index 0000000..1ce5e76 --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiver.kt @@ -0,0 +1,60 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.androidenv.accessibilityforwarder + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.util.Log + +/** Broadcast receiver responsible for enabling or disabling flags. */ +class FlagsBroadcastReceiver() : BroadcastReceiver() { + + override fun onReceive(context: Context?, intent: Intent?) { + val action = intent?.action + Log.i(TAG, "Received broadcast intent with action: " + action) + when (action) { + ACTION_ENABLE_ACCESSIBILITY_TREE_LOGS -> { + Log.i(TAG, "Enabling Accessibility Tree logging.") + LogFlags.logAccessibilityTree = true + } + ACTION_DISABLE_ACCESSIBILITY_TREE_LOGS -> { + Log.i(TAG, "Disabling Accessibility Tree logging.") + LogFlags.logAccessibilityTree = false + } + ACTION_SET_GRPC -> { + // The Android Emulator uses 10.0.2.2 as a redirect to the workstation's IP. Most often the + // gRPC server will be running locally so it makes sense to use this as the default value. + // See https://developer.android.com/studio/run/emulator-networking#networkaddresses. + val host = intent.getStringExtra("host") ?: "10.0.2.2" + // The TCP port to connect. If <=0 gRPC is disabled. + val port = intent.getIntExtra("port", 0) + Log.i(TAG, "Setting gRPC endpoint to ${host}:${port}.") + LogFlags.grpcHost = host + LogFlags.grpcPort = port + } + else -> Log.w(TAG, "Unknown action: ${action}") + } + } + + companion object { + private const val TAG = "FlagsBroadcastReceiver" + private const val ACTION_ENABLE_ACCESSIBILITY_TREE_LOGS = + "accessibility_forwarder.intent.action.ENABLE_ACCESSIBILITY_TREE_LOGS" + private const val ACTION_DISABLE_ACCESSIBILITY_TREE_LOGS = + "accessibility_forwarder.intent.action.DISABLE_ACCESSIBILITY_TREE_LOGS" + private const val ACTION_SET_GRPC = "accessibility_forwarder.intent.action.SET_GRPC" + } +} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiverTest.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiverTest.kt new file mode 100644 index 0000000..f643cfa --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiverTest.kt @@ -0,0 +1,166 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.androidenv.accessibilityforwarder + +import android.content.Intent +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +@RunWith(RobolectricTestRunner::class) +class FlagsBroadcastReceiverTest { + + @Test + fun onReceive_nullIntent_shouldNotLogAnything() { + // Arrange. + LogFlags.logAccessibilityTree = false + val receiver = FlagsBroadcastReceiver() + + // Act. + receiver.onReceive(context = null, intent = null) + + // Assert. + assertThat(LogFlags.logAccessibilityTree).isFalse() + } + + @Test + fun onReceive_nullIntent_actionShouldNotLogAnything() { + // Arrange. + LogFlags.logAccessibilityTree = false + val receiver = FlagsBroadcastReceiver() + val intent = Intent() + + // Act. + receiver.onReceive(context = null, intent = intent) + + // Assert. + assertThat(LogFlags.logAccessibilityTree).isFalse() + } + + @Test + fun onReceive_unknownIntent_actionShouldIssueWarning() { + // Arrange. + LogFlags.logAccessibilityTree = false + val receiver = FlagsBroadcastReceiver() + val intent = Intent("SOME_WEIRD_ACTION") + + // Act. + receiver.onReceive(context = null, intent = intent) + + // Assert. + assertThat(LogFlags.logAccessibilityTree).isFalse() + } + + @Test + fun onReceive_intentWithDisableAction_shouldDisableTreeLogging() { + // Arrange. + LogFlags.logAccessibilityTree = true + val receiver = FlagsBroadcastReceiver() + val intent = Intent("accessibility_forwarder.intent.action.DISABLE_ACCESSIBILITY_TREE_LOGS") + + // Act. + receiver.onReceive(context = null, intent = intent) + + // Assert. + assertThat(LogFlags.logAccessibilityTree).isFalse() + } + + @Test + fun onReceive_intentWithEnableAction_shouldEnableTreeLogging() { + // Arrange. + LogFlags.logAccessibilityTree = false + val receiver = FlagsBroadcastReceiver() + val intent = Intent("accessibility_forwarder.intent.action.ENABLE_ACCESSIBILITY_TREE_LOGS") + + // Act. + receiver.onReceive(context = null, intent = intent) + + // Assert. + assertThat(LogFlags.logAccessibilityTree).isTrue() + } + + @Test + fun onReceive_intentWithSetGrpcActionNoArgs_shouldDefaultToEmuIpAndPortZero() { + // Arrange. + LogFlags.grpcHost = "some_host" + LogFlags.grpcPort = 9999 + val receiver = FlagsBroadcastReceiver() + val intent = Intent("accessibility_forwarder.intent.action.SET_GRPC") + + // Act. + receiver.onReceive(context = null, intent = intent) + + // Assert. + assertThat(LogFlags.grpcHost).isEqualTo("10.0.2.2") + assertThat(LogFlags.grpcPort).isEqualTo(0) + } + + @Test + fun onReceive_intentWithSetGrpcActionWithHostNoPort_shouldDefaultPortToZero() { + // Arrange. + LogFlags.grpcHost = "some_host" + LogFlags.grpcPort = 9999 + val receiver = FlagsBroadcastReceiver() + val intent = + Intent("accessibility_forwarder.intent.action.SET_GRPC").apply { + putExtra("host", "awesome.server.ca") + } + + // Act. + receiver.onReceive(context = null, intent = intent) + + // Assert. + assertThat(LogFlags.grpcHost).isEqualTo("awesome.server.ca") + assertThat(LogFlags.grpcPort).isEqualTo(0) + } + + @Test + fun onReceive_intentWithSetGrpcActionWithPortNoHost_shouldDefaultHostToEmuIp() { + // Arrange. + LogFlags.grpcHost = "some_host" + LogFlags.grpcPort = 9999 + val receiver = FlagsBroadcastReceiver() + val intent = + Intent("accessibility_forwarder.intent.action.SET_GRPC").apply { putExtra("port", 54321) } + + // Act. + receiver.onReceive(context = null, intent = intent) + + // Assert. + assertThat(LogFlags.grpcHost).isEqualTo("10.0.2.2") + assertThat(LogFlags.grpcPort).isEqualTo(54321) + } + + @Test + fun onReceive_intentWithSetGrpcActionWithHostAndPort_shouldSetBoth() { + // Arrange. + LogFlags.grpcHost = "some_host" + LogFlags.grpcPort = 9999 + val receiver = FlagsBroadcastReceiver() + val intent = + Intent("accessibility_forwarder.intent.action.SET_GRPC").apply { + putExtra("host", "grpc.ca") + putExtra("port", 54321) + } + + // Act. + receiver.onReceive(context = null, intent = intent) + + // Assert. + assertThat(LogFlags.grpcHost).isEqualTo("grpc.ca") + assertThat(LogFlags.grpcPort).isEqualTo(54321) + } +} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/LogFlags.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/LogFlags.kt new file mode 100644 index 0000000..7af6fbd --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/LogFlags.kt @@ -0,0 +1,32 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.androidenv.accessibilityforwarder + +/** + * Controls global settings in AccessibilityForwarder. + * + * Please note that this class is not thread safe. + */ +object LogFlags { + // Whether to log the accessibility tree. + var logAccessibilityTree: Boolean = false + // How frequent to emit a11y trees (in milliseconds). + var a11yTreePeriodMs: Long = 100 + + // The gRPC server to connect to. (Only available if grpcPort>0). + var grpcHost: String = "" + // If >0 this represents the gRPC port number to connect to. + var grpcPort: Int = 0 +} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/ParentChildNodePair.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/ParentChildNodePair.kt new file mode 100644 index 0000000..4773a5c --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/ParentChildNodePair.kt @@ -0,0 +1,40 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.androidenv.accessibilityforwarder + +import android.view.accessibility.AccessibilityNodeInfo +import com.google.auto.value.AutoValue + +/** Parent and child [AccessibilityNodeInfo] relationship. */ +@AutoValue +internal abstract class ParentChildNodePair { + abstract fun parent(): AccessibilityNodeInfo? + + abstract fun child(): AccessibilityNodeInfo + + /** [ParentChildNodePair] builder. */ + @AutoValue.Builder + abstract class Builder { + abstract fun parent(parent: AccessibilityNodeInfo?): Builder + + abstract fun child(child: AccessibilityNodeInfo): Builder + + abstract fun build(): ParentChildNodePair + } + + companion object { + @JvmStatic fun builder(): Builder = AutoValue_ParentChildNodePair.Builder() + } +} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/UniqueIdsGenerator.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/UniqueIdsGenerator.kt new file mode 100644 index 0000000..eeedacd --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/UniqueIdsGenerator.kt @@ -0,0 +1,29 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.androidenv.accessibilityforwarder + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.Function + +/** Thread-safe helper class for assigning a unique ID to an object. */ +internal class UniqueIdsGenerator { + private val nextId = AtomicInteger(0) + private val uniqueIdsByNode = ConcurrentHashMap() + + fun getUniqueId(a: A): Int { + return uniqueIdsByNode.computeIfAbsent(a, Function { _: A -> nextId.getAndIncrement() }) + } +} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/res/xml/accessibility_forwarder_service.xml b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/res/xml/accessibility_forwarder_service.xml new file mode 100644 index 0000000..e73943d --- /dev/null +++ b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/res/xml/accessibility_forwarder_service.xml @@ -0,0 +1,21 @@ +// Copyright 2024 DeepMind Technologies Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + + +