Fix/window race around

openadapt/alembic/versions/f9397786028d_add_a11y_event_remove_state_from_window.py

@@ -0,0 +1,57 @@
+"""add_a11y_event_remove_state_from_window
+
+Revision ID: f9397786028d
+Revises: 98505a067995
+Create Date: 2024-10-21 23:15:25.932599
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import sqlite
+import openadapt
+
+# revision identifiers, used by Alembic.
+revision = 'f9397786028d'
+down_revision = '98505a067995'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('replay',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True),
+    sa.Column('strategy_name', sa.String(), nullable=True),
+    sa.Column('strategy_args', sa.JSON(), nullable=True),
+    sa.Column('git_hash', sa.String(), nullable=True),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_replay'))
+    )
+    op.create_table('a11y_event',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True),
+    sa.Column('handle', sa.Integer(), nullable=True),
+    sa.Column('data', sa.JSON(), nullable=True),
+    sa.ForeignKeyConstraint(['handle', 'timestamp'], ['window_event.handle', 'window_event.timestamp'], name=op.f('fk_a11y_event_handle_window_event')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_a11y_event'))
+    )
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('handle', sa.Integer(), nullable=True))
+        batch_op.create_unique_constraint('uix_handle_timestamp', ['handle', 'timestamp'])
+        batch_op.drop_column('window_id')
+        batch_op.drop_column('state')
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('state', sqlite.JSON(), nullable=True))
+        batch_op.add_column(sa.Column('window_id', sa.VARCHAR(), nullable=True))
+        batch_op.drop_constraint('uix_handle_timestamp', type_='unique')
+        batch_op.drop_column('handle')
+
+    op.drop_table('a11y_event')
+    op.drop_table('replay')
+    # ### end Alembic commands ###

openadapt/app/dashboard/api/recordings.py

@@ -11,6 +11,7 @@
 from openadapt.models import Recording
 from openadapt.plotting import display_event
 from openadapt.utils import image2utf8, row2dict
+from openadapt.visualize import dict2html
 
 
 class RecordingsAPI:
@@ -112,6 +113,8 @@ def convert_to_str(event_dict: dict) -> dict:
 
             for action_event in action_events:
                 event_dict = row2dict(action_event)
+                a11y_dict = row2dict(action_event.window_event.a11y_event)
+                event_dict["a11y_data"] = dict2html(a11y_dict)
                 try:
                     image = display_event(action_event)
                     width, height = image.size

openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx

@@ -41,7 +41,7 @@ export const ActionEvent = ({
     let content = (
         <Grid align='center'>
             <Grid.Col span={8}>
-                <Table w={400} withTableBorder withColumnBorders my={20} className='border-2 border-gray-300 border-solid'>
+                <Table withTableBorder withColumnBorders my={20} className='border-2 border-gray-300 border-solid'>
                     <Table.Tbody>
                         {typeof event.id === 'number' && (
                             <TableRowWithBorder>
@@ -65,12 +65,6 @@ export const ActionEvent = ({
                                 <TableCellWithBorder>{timeStampToDateString(event.screenshot_timestamp)}</TableCellWithBorder>
                             </TableRowWithBorder>
                         )}
-                        {event.browser_event_timestamp && (
-                            <TableRowWithBorder>
-                                <TableCellWithBorder>browser event timestamp</TableCellWithBorder>
-                                <TableCellWithBorder>{timeStampToDateString(event.browser_event_timestamp)}</TableCellWithBorder>
-                            </TableRowWithBorder>
-                        )}
                         <TableRowWithBorder>
                             <TableCellWithBorder>window event timestamp</TableCellWithBorder>
                             <TableCellWithBorder>{timeStampToDateString(event.window_event_timestamp)}</TableCellWithBorder>
@@ -142,6 +136,50 @@ export const ActionEvent = ({
                     </Table.Tbody>
                 </Table>
             </Grid.Col>
+            {event.a11y_data && (
+                <Grid.Col span={12}>
+                        <Accordion className="w-full">
+                            <Accordion.Item value="a11y_data">
+                                <Accordion.Control>
+                                    <div className="text-blue-500 w-full cursor-pointer">
+                                        Accessibility Data
+                                    </div>
+                                </Accordion.Control>
+                                <Accordion.Panel>
+                                    <Table  withTableBorder withColumnBorders my={20} className="border-2 border-gray-300 border-solid w-full">
+                                        <Table.Tbody>
+                                            <style>
+                                                {`
+                                                    .table-nested table {
+                                                    border-collapse: collapse;
+                                                    border: 1px solid #d1d5db;
+                                                    }
+
+                                                    .table-nested th,
+                                                    .table-nested td {
+                                                    border: 1px solid #d1d5db;
+                                                    padding: 8px;
+                                                    text-align: left;
+                                                    }
+
+                                                    .table-nested th {
+                                                    background-color: #f9fafb;
+                                                    font-weight: bold;
+                                                    }
+
+                                                    .table-nested td {
+                                                    background-color: #ffffff;
+                                                    }
+                                                `}
+                                            </style>
+                                            <div className="table-nested w-full" dangerouslySetInnerHTML={{ __html: event.a11y_data }} />
+                                        </Table.Tbody>
+                                    </Table>
+                                </Accordion.Panel>
+                            </Accordion.Item>
+                        </Accordion>   
+                </Grid.Col>
+            )}
             <Grid.Col span={4}>
                 <RemoveActionEvent event={event} />
             </Grid.Col>

openadapt/app/dashboard/types/action-event.ts

@@ -6,6 +6,7 @@ export type ActionEvent = {
     screenshot_timestamp?: number;
     window_event_timestamp: number;
     browser_event_timestamp: number;
+    a11y_data: string;
     mouse_x: number | null;
     mouse_y: number | null;
     mouse_dx: number | null;

openadapt/config.py

@@ -135,7 +135,7 @@ class SegmentationAdapter(str, Enum):
 
     # Record and replay
     EVENT_BUFFER_QUEUE_SIZE: int = 100
-    RECORD_WINDOW_DATA: bool = True
+    RECORD_A11Y_DATA: bool = True
     RECORD_READ_ACTIVE_ELEMENT_STATE: bool
     RECORD_VIDEO: bool
     RECORD_AUDIO: bool

openadapt/db/crud.py

@@ -28,6 +28,7 @@
     Screenshot,
     ScrubbedRecording,
     WindowEvent,
+    A11yEvent,
     copy_sa_instance,
 )
 from openadapt.privacy.base import ScrubbingProvider
@@ -287,6 +288,25 @@ def insert_recording(session: SaSession, recording_data: dict) -> Recording:
     return db_obj
 
 
+def insert_a11y_event(
+    session: SaSession,
+    event_data: dict,
+) -> None:
+    """Insert an a11y event into the database.
+
+    Args:
+        session (sa.orm.Session): The database session.
+        event_data (dict): The data of the event
+    """
+    handle = event_data["handle"]
+    a11y_data = event_data["a11y_data"]
+    timestamp = event_data["timestamp"]
+    a11y_event = A11yEvent(timestamp=timestamp, handle=handle, data=a11y_data)
+
+    session.add(a11y_event)
+    session.commit()
+
+
 def delete_recording(session: SaSession, recording: Recording) -> None:
     """Remove the recording from the db.
 
@@ -615,6 +635,35 @@ def get_window_events(
     )
 
 
+def get_a11y_events(
+    session: SaSession,
+    recording: Recording,
+) -> list[A11yEvent]:
+    """Get accessibility events for a given recording.
+
+    Args:
+        session (SaSession): The SQLAlchemy session.
+        recording (Recording): The recording object.
+
+    Returns:
+        list[A11yEvent]: A list of accessibility events for the recording.
+    """
+    return (
+        session.query(A11yEvent)
+        .join(
+            WindowEvent,
+            (A11yEvent.handle == WindowEvent.handle)
+            & (A11yEvent.timestamp == WindowEvent.timestamp),
+        )
+        .filter(WindowEvent.recording_id == recording.id)
+        .options(
+            joinedload(A11yEvent.window_event).joinedload(WindowEvent.recording),
+        )
+        .order_by(A11yEvent.timestamp)
+        .all()
+    )
+
+
 def get_browser_events(session: SaSession, recording: Recording) -> list[BrowserEvent]:
     """Get browser events for a given recording.
 

openadapt/events.py

@@ -46,6 +46,7 @@ def get_events(
     action_events = crud.get_action_events(db, recording)
     window_events = crud.get_window_events(db, recording)
     browser_events = crud.get_browser_events(db, recording)
+    a11y_events = crud.get_a11y_events(db, recording)
     screenshots = crud.get_screenshots(db, recording)
 
     browser_stats = browser.assign_browser_events(db, action_events, browser_events)
@@ -67,11 +68,13 @@ def get_events(
     num_window_events = len(window_events)
     num_screenshots = len(screenshots)
     num_browser_events = len(browser_events)
+    num_a11y_events = len(a11y_events)
 
     num_action_events_raw = num_action_events
     num_window_events_raw = num_window_events
     num_screenshots_raw = num_screenshots
     num_browser_events_raw = num_browser_events
+    num_a11y_events_raw = num_a11y_events
     duration_raw = action_events[-1].timestamp - action_events[0].timestamp
 
     num_process_iters = 0
@@ -83,30 +86,35 @@ def get_events(
                 f"{num_window_events=} "
                 f"{num_screenshots=}"
                 f"{num_browser_events=}"
+                f"{num_a11y_events=}"
             )
             (
                 action_events,
                 window_events,
                 screenshots,
                 browser_events,
+                a11y_events,
             ) = merge_events(
                 action_events,
                 window_events,
                 screenshots,
                 browser_events,
+                a11y_events,
             )
             if (
                 len(action_events) == num_action_events
                 and len(window_events) == num_window_events
                 and len(screenshots) == num_screenshots
                 and len(browser_events) == num_browser_events
+                and len(a11y_events) == num_a11y_events
             ):
                 break
             num_process_iters += 1
             num_action_events = len(action_events)
             num_window_events = len(window_events)
             num_screenshots = len(screenshots)
             num_browser_events = len(browser_events)
+            num_a11y_events = len(a11y_events)
             if num_process_iters == MAX_PROCESS_ITERS:
                 break
 
@@ -131,6 +139,10 @@ def get_events(
             num_browser_events,
             num_browser_events_raw,
         )
+        meta["num_a11y_events"] = format_num(
+            num_a11y_events,
+            num_a11y_events_raw,
+        )
 
         duration = action_events[-1].timestamp - action_events[0].timestamp
         if len(action_events) > 1:
@@ -823,17 +835,20 @@ def merge_events(
     window_events: list[models.WindowEvent],
     screenshots: list[models.Screenshot],
     browser_events: list[models.BrowserEvent],
+    a11y_events: list[models.A11yEvent],
 ) -> tuple[
     list[models.ActionEvent],
     list[models.WindowEvent],
     list[models.Screenshot],
+    list[models.A11yEvent],
 ]:
     """Merge redundant action events, window events, and screenshots.
 
     Args:
         action_events (list): The list of action events.
         window_events (list): The list of window events.
         screenshots (list): The list of screenshots.
+        a11y_events (list): The list of a11y events.
 
     Returns:
         tuple: A tuple containing the processed action events, window events,
@@ -843,13 +858,19 @@ def merge_events(
     num_window_events = len(window_events)
     num_screenshots = len(screenshots)
     num_browser_events = len(browser_events)
+    num_a11y_events = len(a11y_events)
     num_total = (
-        num_action_events + num_window_events + num_screenshots + num_browser_events
+        num_action_events
+        + num_window_events
+        + num_screenshots
+        + num_browser_events
+        + num_a11y_events
     )
     logger.info(
         "before"
         f" {num_action_events=} {num_window_events=}"
         f" {num_screenshots=} {num_browser_events=} "
+        f"{num_a11y_events=} "
         f"{num_total=}"
     )
     process_fns = [
@@ -893,28 +914,40 @@ def merge_events(
             action_events,
             "browser_event_timestamp",
         )
+        a11y_events = discard_unused_events(
+            a11y_events,
+            action_events,
+            "timestamp",
+        )
     num_action_events_ = len(action_events)
     num_window_events_ = len(window_events)
     num_screenshots_ = len(screenshots)
     num_browser_events_ = len(browser_events)
+    num_a11y_events_ = len(a11y_events)
     num_total_ = (
-        num_action_events_ + num_window_events_ + num_screenshots_ + num_browser_events_
+        num_action_events_
+        + num_window_events_
+        + num_screenshots_
+        + num_browser_events_
+        + num_a11y_events_
     )
     pct_action_events = num_action_events_ / num_action_events
     pct_window_events = num_window_events_ / num_window_events
     pct_screenshots = num_screenshots_ / num_screenshots
     pct_browser_events = (
         num_browser_events_ / num_browser_events if num_browser_events else None
     )
+    pct_a11y_events = num_a11y_events_ / num_a11y_events
     pct_total = num_total_ / num_total
     logger.info(
         "after"
         f" {num_action_events_=} {num_window_events_=}"
         f" {num_screenshots_=} {num_browser_events_=}"
+        f"{num_a11y_events_=} {num_a11y_events_=} "
         f" {num_total_=}"
     )
     logger.info(
         f"{pct_action_events=} {pct_window_events=} {pct_screenshots=}"
-        f" {pct_browser_events=} {pct_total=}"
+        f" {pct_browser_events=} {pct_a11y_events=} {pct_total=}"
     )
-    return action_events, window_events, screenshots, browser_events
+    return action_events, window_events, screenshots, browser_events, a11y_events