How to get touch coordinates?

[zhuraromdev]

Hello, how can I store coordinates of mouse?

I was trying to make like this, however numbers seems to be wrong:

def get_touch_coordinates():
    touch_coordinates = []
    process = subprocess.Popen(['adb', 'shell', 'getevent', '-lt'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    for line in process.stdout:
        line = line.decode('utf-8').strip()
        if 'ABS_MT_POSITION_X' in line or 'ABS_MT_POSITION_Y' in line:
            coords = line.split()[-1]
            touch_coordinates.append(int(coords, 16))
            if len(touch_coordinates) == 2:
                x, y = touch_coordinates
                touch_coordinates = []
                return x, y

Example of output:

timestamp,event_type,x,y,button,screenshot_path,screen_width,screen_height,application_name,trajectory
2024-08-06_14-49-11_281052,click,3623,8631,,and/screenshot_2024-08-06_14-49-11_281052.png,2208,1840,and,
2024-08-06_14-49-13_687736,click,4007,9671,,and/screenshot_2024-08-06_14-49-13_687736.png,2208,1840,and,
2024-08-06_14-49-16_482075,click,4167,5807,,and/screenshot_2024-08-06_14-49-16_482075.png,2208,1840,and,
2024-08-06_14-49-19_139051,click,2807,9719,,and/screenshot_2024-08-06_14-49-19_139051.png,2208,1840,and,
2024-08-06_14-49-21_497052,click,3839,13479,,and/screenshot_2024-08-06_14-49-21_497052.png,2208,1840,and,

Also sending the full code, for better understanding:

from PIL import Image, ImageDraw
import datetime
import os
import pandas as pd
import threading
import adbutils
import subprocess

# Connect to Android emulator
def connect_device():
    adb = adbutils.AdbClient(host="127.0.0.1", port=5037)
    devices = adb.device_list()
    if not devices:
        raise adbutils.errors.AdbError("Can't find any android device/emulator")
    return devices[0]

try:
    device = connect_device()
except adbutils.errors.AdbError as e:
    print(e)
    print("Ensure your emulator is running and connected.")
    exit(1)

# Initialize a DataFrame to store logs
df_columns = ["timestamp", "event_type", "x", "y", "button", "screenshot_path", "screen_width", "screen_height", "application_name", "trajectory"]
df = pd.DataFrame(columns=df_columns)

# Function to get touch coordinates from the emulator
def get_touch_coordinates():
    touch_coordinates = []
    process = subprocess.Popen(['adb', 'shell', 'getevent', '-lt'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    for line in process.stdout:
        line = line.decode('utf-8').strip()
        if 'ABS_MT_POSITION_X' in line or 'ABS_MT_POSITION_Y' in line:
            coords = line.split()[-1]
            touch_coordinates.append(int(coords, 16))
            if len(touch_coordinates) == 2:
                x, y = touch_coordinates
                touch_coordinates = []
                return x, y

# Function to get the screen size of the emulator
def get_screen_size():
    output = subprocess.check_output(['adb', 'shell', 'wm', 'size']).decode('utf-8')
    dimensions = output.split()[-1]
    width, height = map(int, dimensions.split('x'))
    return width, height

# Function to log the event and take a screenshot from Android emulator
def log_and_screenshot(event_type, x, y, button=None):
    timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S_%f")
    screenshot_path = os.path.join(screenshot_dir, f"screenshot_{timestamp}.png")

    # Capture screenshot from Android emulator
    temp_screenshot_path = "/sdcard/screenshot.png"
    device.shell(['screencap', '-p', temp_screenshot_path])
    device.sync.pull(temp_screenshot_path, screenshot_path)
    device.shell(['rm', temp_screenshot_path])

    screenshot = Image.open(screenshot_path)
    draw = ImageDraw.Draw(screenshot)
    cursor_size = 10

    # Scale the raw touch coordinates to match the screen size
    x = int(x * screen_width / 4096)  # Adjust based on your device resolution and maximum coordinate range
    y = int(y * screen_height / 4096)

    # Draw the cursor position with a green cross
    draw.line((x - cursor_size, y, x + cursor_size, y), fill="green", width=2)
    draw.line((x, y - cursor_size, x, y + cursor_size), fill="green", width=2)

    screenshot.save(screenshot_path, format='PNG')

    # Log the event
    event_data = {
        "timestamp": timestamp,
        "event_type": event_type,
        "x": x,
        "y": y,
        "button": button,
        "screenshot_path": screenshot_path,
        "screen_width": screen_width,
        "screen_height": screen_height,
        "application_name": application_name,
        "trajectory": ""
    }
    global df
    df = pd.concat([df, pd.DataFrame([event_data])], ignore_index=True)
    df.to_csv(log_file_csv, index=False)

    with open(log_file_txt, "a") as f:
        f.write(f"{timestamp} - {event_type} - ({x}, {y}) - {button} - {screenshot_path} - {screen_width}x{screen_height} - {application_name}\n")

# Function to listen for touch events and log them
def listen_for_touch_events():
    while True:
        coordinates = get_touch_coordinates()
        if coordinates:
            x, y = coordinates
            log_and_screenshot("click", x, y)

# Main function to handle user choice
def main():
    global application_name, screenshot_dir, log_file_csv, log_file_txt, screen_width, screen_height
    application_name = input("Enter the application, which you want to record: ")

    # Create a new directory for the screenshots based on the application name
    screenshot_dir = application_name
    os.makedirs(screenshot_dir, exist_ok=True)

    # Define the log files in the new directory
    log_file_csv = os.path.join(screenshot_dir, "mouse_log.csv")
    log_file_txt = os.path.join(screenshot_dir, "mouse_log.txt")

    # Get screen size from the Android emulator
    screen_width, screen_height = get_screen_size()

    print("Choose an option:")
    print("1. Automatic mode")
    print("2. Manual mode")
    choice = input("Enter your choice (1 or 2): ")

    if choice == '1':
        # Start performing random actions in a separate thread
        threading.Thread(target=perform_random_actions).start()
    elif choice == '2':
        # Start listening for touch events
        threading.Thread(target=listen_for_touch_events, daemon=True).start()
        print("Manual mode activated. Click in the emulator to log events.")
    else:
        print("Invalid choice. Exiting.")
        return

    # Keep the script running to listen for events
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        print("Script terminated.")

if __name__ == "__main__":
    main()